{"meta":{"start_date":"2026-08-24","end_date":"2026-08-30","start_at":"2026-08-24T00:00:00Z","end_at":"2026-08-31T00:00:00Z","timezone":"UTC","generated_at":"2026-08-25T22:05:14Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":240,"failed_runs":213,"raw_occurrences":863,"matched_occurrences":817,"runs_affected":167},"rows":[{"environment":"dev","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":116,"runs_affected":56,"failed_runs":56,"run_impact_percent":23.333333333333332,"anchor_occurrences":116,"after_last_push_count":22,"also_in":["INT"],"trend_counts":[0,31,85,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":89,"prior_last_seen_at":"2026-08-22T18:55:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":12},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":10},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":8},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":13},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":11},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--8n2g57dcg9db/cluster-hs-pre-gvdq5r should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005deea0\u003e: \n failed to create HCP cluster cluster-hs-pre-gvdq5r: failed waiting for cluster=\"cluster-hs-pre-gvdq5r\" in resourcegroup=\"rg-hs-pre--8n2g57dcg9db\" to finish creating: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/471e2801-5a5e-4e50-bf64-e6463da799d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/471e2801-5a5e-4e50-bf64-e6463da799d2\",\n \"name\": \"471e2801-5a5e-4e50-bf64-e6463da799d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:22:38.686763897Z\",\n \"endTime\": \"2026-08-25T20:41:43.170399115Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--8n2g57dcg9db/cluster-hs-pre-gvdq5r should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005deea0\u003e: \n failed to create HCP cluster cluster-hs-pre-gvdq5r: failed waiting for cluster=\"cluster-hs-pre-gvdq5r\" in resourcegroup=\"rg-hs-pre--8n2g57dcg9db\" to finish creating: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/471e2801-5a5e-4e50-bf64-e6463da799d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/471e2801-5a5e-4e50-bf64-e6463da799d2\",\n \"name\": \"471e2801-5a5e-4e50-bf64-e6463da799d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:22:38.686763897Z\",\n \"endTime\": \"2026-08-25T20:41:43.170399115Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001588640\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7xsx844dgk5p\" to finish creating: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/82d6a6fb-35cc-4769-b1cb-973346fa5bcc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/82d6a6fb-35cc-4769-b1cb-973346fa5bcc\",\n \"name\": \"82d6a6fb-35cc-4769-b1cb-973346fa5bcc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:22:34.356941864Z\",\n \"endTime\": \"2026-08-25T20:41:43.091321632Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001588640\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7xsx844dgk5p\" to finish creating: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/82d6a6fb-35cc-4769-b1cb-973346fa5bcc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/82d6a6fb-35cc-4769-b1cb-973346fa5bcc\",\n \"name\": \"82d6a6fb-35cc-4769-b1cb-973346fa5bcc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:22:34.356941864Z\",\n \"endTime\": \"2026-08-25T20:41:43.091321632Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-2ktt644zvj5d/cluster-candidate-5-0-sz2cpd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011e0260\u003e: \n failed to create HCP cluster cluster-candidate-5-0-sz2cpd: failed waiting for cluster=\"cluster-candidate-5-0-sz2cpd\" in resourcegroup=\"rg-candidate-5-0-2ktt644zvj5d\" to finish creating: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9119e2e-12f1-4cf3-850d-92df0e16d5a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9119e2e-12f1-4cf3-850d-92df0e16d5a4\",\n \"name\": \"d9119e2e-12f1-4cf3-850d-92df0e16d5a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:22:49.973596288Z\",\n \"endTime\": \"2026-08-25T20:41:53.170244767Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-2ktt644zvj5d/cluster-candidate-5-0-sz2cpd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011e0260\u003e: \n failed to create HCP cluster cluster-candidate-5-0-sz2cpd: failed waiting for cluster=\"cluster-candidate-5-0-sz2cpd\" in resourcegroup=\"rg-candidate-5-0-2ktt644zvj5d\" to finish creating: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9119e2e-12f1-4cf3-850d-92df0e16d5a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9119e2e-12f1-4cf3-850d-92df0e16d5a4\",\n \"name\": \"d9119e2e-12f1-4cf3-850d-92df0e16d5a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:22:49.973596288Z\",\n \"endTime\": \"2026-08-25T20:41:53.170244767Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092336210943938560","occurred_at":"2026-08-25T19:39:54Z","signature_id":"6a76f01db033a3c0dcfb388eccd60792d5b0d41d188f1e54f33061f14ee502f1","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092336210943938560","occurred_at":"2026-08-25T19:39:54Z","signature_id":"c2c799804a88320b88fb4080134a18106ed5a0a378000d4530a27d4d51efecb5","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092336210943938560","occurred_at":"2026-08-25T19:39:54Z","signature_id":"c7b43690396d7677f1e8f1ebea7156f74a59bbbb2ff5ebde9d881e6e5232f7b1","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092336210943938560","occurred_at":"2026-08-25T19:39:54Z","signature_id":"dc4094111491512d4c0555b5c9454be3706e7e79856df97bc7a80bde412714e3","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092332915538006016","occurred_at":"2026-08-25T19:26:55Z","signature_id":"8f9111de818db6ad374131758f5c00285ad5268f2bfe07668b44a478f919cd34","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092331734132592640","occurred_at":"2026-08-25T19:22:13Z","signature_id":"26e428d2c9c6aae96b320dd31ccf6f4910c7744b098354a2cb00c69fcaabb11f","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092326377335296000","occurred_at":"2026-08-25T19:00:56Z","signature_id":"170757c76139104c8b8ef676295851ee3d3a9f125544e268f5c10895238d0936","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092326377335296000","occurred_at":"2026-08-25T19:00:56Z","signature_id":"48ad0ea618c58e4a30949aaba38a2b987f7a40ff70cd06c1535d7ecc67b033b4","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092326377335296000","occurred_at":"2026-08-25T19:00:56Z","signature_id":"ab6ea526ae58270b81a9565789b7a538e82280d9f1764435fda9ffb09b16196f","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314723616821248","occurred_at":"2026-08-25T18:14:37Z","signature_id":"0076fbe063547b07110db9c853686695f6d9b45408e17004b5c20f9f45967910","pr_number":6682,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314723616821248","occurred_at":"2026-08-25T18:14:37Z","signature_id":"77c50bc17b9d53d03c763973e2fb5dbf388cfdf6e7d1846e1e64d2e0d6d3a12d","pr_number":6682,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"58bf9ab1eb1b504ee63dc64568b36cd225e7db6ef55de533f9e062770f4ac7f7","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"d9c26ef58d5fcf8c49cf5a4e45b95fc06f0f9d202df046f2ea975e15fe32ba90","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"f72cda8e54add5e222bb916eb78bd6882ff8c549adafd27df57b6813b5fa6eee","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"f731da949e2cbc194b5598178bee5be2ece9258202ff2948a41a7f722a5d15b6","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","signature_id":"08d84795c86bdd39c92b3ef7ed4171353ac5d7a95629fb6767e9d5e53dab0575","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","signature_id":"09f8cbe70cbae7527497ee988bb4e3ca9bd72951deeb0cf5d80d73cf512c0032","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","signature_id":"87d15fc3b3d51f2f525f669b15efa9810a7a54c78626d39624a2206eb8b40682","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","signature_id":"f7690a17f140753a646175aaf07b86c6525354b0ca8e94c3547454592f5e4408","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","signature_id":"fc8cdd9240844df1f37ddc400bdd2c5a950866210e7f77cce8d91f909ab5d0c9","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","occurred_at":"2026-08-25T16:56:19Z","signature_id":"0669947174a86499d5670bc164ea47f8372430e063dda27921fe41db635f2d1a","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","occurred_at":"2026-08-25T16:56:19Z","signature_id":"ca1da36a392a93e6f3a6490b4497374f23e1bdcba26c42be24847ece14ed3f61","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284063539793920","occurred_at":"2026-08-25T16:12:48Z","signature_id":"b168e3a1107e3e1902272b9ff4556d75fd8dfe44f624dfd2607bc5f72bb938f0","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284063539793920","occurred_at":"2026-08-25T16:12:48Z","signature_id":"f71dae65ad1aafde8d4f6ae00fb97c69b8fec2ec7c262dff3ab35decf7d7de86","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284042668937216","occurred_at":"2026-08-25T16:12:43Z","signature_id":"81a27532541d67e6137f3b18dfa24f47172a110e72475bc734463343f3f6e146","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284042668937216","occurred_at":"2026-08-25T16:12:43Z","signature_id":"87fa0e9ed440b5075b8c246f9d4ac0dfb3a65d6379a21ec17eac51d7fe33d902","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092283360171790336","occurred_at":"2026-08-25T16:10:00Z","signature_id":"e091e7c3963b1d439bcf214121c57fa63b1447fa3dcc926afb3e313210a13c5e","pr_number":6630,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"2d44f9e1ef87d22b647e496ad891393923d8b6071781ac4f3325409453b16aec","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"431183966ffec9d316eabffd849e7f6e0b0f56934e37ad7b18ebee3f9d0c7785","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"6d0a93471cca29b596f2665a2e4b14d8f14b38ab392d8cc47699b7365343a0d3","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"a34616cf01ee0e699438c4845004c7e68738e05288f74d735ab18aa5dbd40940","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"aa50862b2be22c2ff41532ef6198e1092baabb54df9335776d5e11eaa6a79bd4","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"4f01337717137a4b91f0a8939b1c66d8d9f1d513029bbfdaa010a32c379be1fb","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"9a5de514cbd74ec8858c418a7e3bc6208cf5c8a2a6c4d616b5ae583b533e5e79","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","signature_id":"7f19b237d650c3e2c2dada758b0faf353ceddfaf46bd9c7e365fe5dd0616b720","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","signature_id":"87ca9ce38c81638d67c18762111fbc84a3f902586128fffc4d6a63e5d5d586d7","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","signature_id":"c301bcb0ad350b3e4edb18ab6b3112a4e915bcc1ccff4534d5bef4c12af07f76","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","occurred_at":"2026-08-25T14:43:04Z","signature_id":"8570bc7030e47ad260a5bdf4a8d1e6ec729b76caed85d2d85acd384bcef9aaf5","pr_number":6676,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","occurred_at":"2026-08-25T14:43:04Z","signature_id":"adc58fbe078ffd0b9a7c1502fc926ccd76f020ceef6eb743ef3c165775530e5a","pr_number":6676,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"2326e62e81a715665c83cf8208b5b992003fcc3c884e154ce68d5a2525a2a304","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"a0a72a832973622295dca8311edc8acdd0d1be0a93774aa5a3c2c41f0a225677","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"b5155d1abd3afd29cd7d2bb03f43d6d1523d798062871307fb209c787f593dc6","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"bd2bd12402128b3e6c7b2c7dc8891fd4bb2cbbd5fb1f6cc30af3b1c2b3e2de09","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"ca98a15fd3dbeb3c9c2ada441d28414afe433e197fe77478335816c4c14eb15a","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","signature_id":"40844011a6d47c6fdea97725589c77bfe98a0d94d59281c86c002f1a0792c273","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253504889425920","occurred_at":"2026-08-25T14:11:22Z","signature_id":"4d865de5fe9dfc533ed0216b192b218fb0191025036e4ded553b013e74cb84a3","pr_number":6587,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"0dd4ca7c298b5c4afeaab0da7cacaeb92e7380f6b13a23bb92ff6fc133819fd1","pr_number":6630,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","signature_id":"02ce83b411d6d3eadebb040a79dc7e0fc19f5dc8037f2c40f8fbbf51c121a5e0","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","signature_id":"59eb8e67410e16bbb2ec63be568c198772ad94ee9f1c1cbda4a99f3c7987422a","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","signature_id":"7bd5b0bf2a08febff61bb7112c5fa60448aafb6f03c45e0ecefda98be25fcc90","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","signature_id":"c1c5c1541665e5683a94093cd7f6588bce5f0027f1e226c12d34e5a06ef243a7","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","signature_id":"0fc6eaf247bde243614d205434c32f36a568acb9c7fa2017a78c81ce9292cbf1","pr_number":6677,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","signature_id":"ebdc0b91a773f55107daffc829136d740ef7567c97d135876b3ab2f729e95efe","pr_number":6677,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"139f5384ed9c8675144c62f1bbdf582bf6ba3de5a1a2a7e69cafbaf9e1324971","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"3bedaad2d7a72d761e92c1a0c1322773388504505b8f7b830d9c172d097aab52","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"4a2ed9f21c01469e954cb46911f58a6903c7bef2bb37ab47e1b4136d17c074f9","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"52e3a2d01cede408cbe7165d526befd88f9e9f4dbcdc8aaa12bb61402619c205","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"8bd3d86b83f9075e0c109be0d76439a6735162556e0e3886ba813816b230c868","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"abc8216563c683da1f6e16ae575f37dd3f305f261e135e096e5da20727d96771","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"f096e2894e5607754bc52d3997abbbaf63ee7b84458d5fadbc6b046423af77c3","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092231268828712960","occurred_at":"2026-08-25T12:43:00Z","signature_id":"88ef5278beac4e0b8897706c7e61f4cbb2e358d71c83d41a4ebf5e0e7c99a5ec","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092231268828712960","occurred_at":"2026-08-25T12:43:00Z","signature_id":"c3e9e8fbb2c80bbe0532378e51b9ba35394e8a18321158765e0e9d2c9531e792","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092231268828712960","occurred_at":"2026-08-25T12:43:00Z","signature_id":"cba54857857016b020e86b6a86de63b4978237a179d12aeabfff1cca95a87fa0","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197879035727872","occurred_at":"2026-08-25T10:30:20Z","signature_id":"ba4575d0d05801721860d214c9ab20325f6c0ce684b7a86f371bbdb0b3c221f6","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"30b876ac66882fc0e335572b0826b8ec58b54d7e36dc27655838b99b58cd8eb0","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"58a3af70c95169637c5e6f6aa6a771a555147df7756a6c67c6c7a78311b59eee","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"59336122fa10346af3e78ba303a4d90407e92e02b272f6b3005e01628acb96b4","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"79c7c41c97eedc396d88101d613a41fb1d71de25afa00931594fcc8ea799c229","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"938d48f5f5615798cf28a4204904814739ed9c8d4708bcdf50c0a0e5b416c33a","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"16c6bbce9d00cd3fdb259dff38344220eb8bc09eaf6c7024a6b9039971482f59","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146681486249984","occurred_at":"2026-08-25T07:06:53Z","signature_id":"e77ed3272a3aa5a99a10d9c5ad6121981ac3eb73b228a8dd38fba0c6882dbb91","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092142908357480448","occurred_at":"2026-08-25T06:51:54Z","signature_id":"180ef015a082a3e691d738ac631464aaaf83f5b0ebf48a41b3ee60048cb89ac9","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"2c553608c906b4a9046fc287e40a200fb56683ae155754190ceb3113e9c01826","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"926bbbc4110caa69d86b22306ab7a9ad88bd051e3a6201884a30ef9e28c59b9b","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"df56cc165c01e6c8c810f48db72eecd1abb1ce36621379d823c8e5cb3996a5ed","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"f9e5e4b9fd79fc6bdafcb10977e537bfeabb8f238b48fc177d8dc722a58ba115","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","signature_id":"8d4d063103cc70a30037cefb230c7659ae5f161bfbe5aceb58476f2a4134c515","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","signature_id":"9ccbbc0b72f96cf97c329b914d4f5cc67ebf262aaacfa9b2db48024f1cbc3920","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","signature_id":"a71e16e9452ebbcb96b7843af9a8f55da51d63b8966fc237dc151347da86709b","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","signature_id":"82fe3afdd0fba60d3c7679d4a315c8480ecd5cf974435df02861bd04baa9a825","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","signature_id":"ac5746c88f40ad3ccf73d0e708c6422fe20ba547abaaaef12f47d1492ae2b838","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"7f995c509ce4acadbbdef98d81915a02d692abceffa2a81b563b98b98cf85466","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","occurred_at":"2026-08-25T00:21:02Z","signature_id":"83e3baeadd89b998c4b81d1bc0c70d07aa9fec97f2694533c8ed2432ea266d46","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","occurred_at":"2026-08-25T00:21:02Z","signature_id":"983555e8ae88df4ba006bdd3a98018885a3dc5949799b00108d57edd8321b6fb","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","occurred_at":"2026-08-25T00:21:02Z","signature_id":"f1a6c7858d28eeb824a65797f9f24cbd2dea977f3e661a9e3c7f104a3dc601ee","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","signature_id":"54a530c9876fc008fec6ac6899f443f3e3f5628a64422efb1e6064e8186af6e1","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","signature_id":"e6a9db1c5a3cafd8f545d8476a1e6b49dd9166a98aa20d59709936d40b1c52c0","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092019318316339200","occurred_at":"2026-08-24T22:40:47Z","signature_id":"a6a8523a32b9fba91a90fbd04abe90465e11b86cdf06949b929a37a85e4f40ad","pr_number":6663,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092019318316339200","occurred_at":"2026-08-24T22:40:47Z","signature_id":"e6285c4c12e9e0c85b5ecc25058de40b9f05b37d1a08c8a8c4a66108766b8ee0","pr_number":6663,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","signature_id":"4549c28d9f346a4101b2d199c56363a0a36787b6952f32440182d5210d9903d4","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","signature_id":"916b71cbe1c0282bebb7706c18cf185456f94291c6c7059f8d673bee6920e761","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"49911eb59aa73a3d7ebf02bdb4e56de7a616897c72e3ac7abe420c0c7a47e40e","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"6487190a1499ee6eb043aa40a245add60c110f31922a104a2b7ccae5c75948f8","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"7cc1252e9b8fe966bbe77836b76cbb84f92a735b546bfadc6681cd7399872b22","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"a9397a1f0aaba5d8ad982fe383aca1267736bcbd0d40f55741f427e85ee8b6d8","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"c5d8b0e5c56e2c31565dc9e53627bd3252420239c844111c2d9db849f9e9c51c","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"deae4888a32973190e55fe6f1f2ac8a8941dd6efae21aa17eaa50cc4448f5b70","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"7451289a87442818c6daa03144d0fb9efebafa88d3687d31f99e8439ca741e97","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","signature_id":"7830c998ab91a0251740ef30429a36f78ee908a1781d172a1f172ebef622c56d","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963250454302720","occurred_at":"2026-08-24T18:58:00Z","signature_id":"75d7cd315bc777c1079b9374c7a2b4aabd61d42a04817f2523f7ea7dab1326a9","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"2021d27012220f14a5eaeb1e574cf0e0cf00596ef715d56d7ab033d1204d0ed5","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"dea6785389a59c2a975d6ddfe69a85996e881ce704015e7a9c41548aeaca0017","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"19ba2327e177a466c74ef48f9d128288b644237f0c8199be7309ae0bf40f5a82","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932257060130816","occurred_at":"2026-08-24T16:54:39Z","signature_id":"70c12268048fb1a990e1fcfe48d8d53530e6791148ebcbe91068d54355e50247","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"688dcfad091ac3bb942f27fcd5e2e3d6471f49c9042338c1c965ab6c339ae7e6","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"d1005ca6dc326e3c744c86ae1e9fb1086de2e12e4a6351183145527b31e5eccf","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"db07fb00cb7c1dc1d49041e7a151eab39b4fee2bc9901dd00a1fc182625826d2","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"40b147b3a09185f7fbbd5416adf8d0e158c67796d5387c2aa465c0a43fd1dc30","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"8d53ed6b1258502dfa6cecd4e6049f72772418b7686ae5c9e29c6adc65e73c79","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"fd285a3626d258dcb1ba2d99b9eb4762dc5e5af32183bf400bd799c872dd2fda","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"94f9ba782dfa48d054a8eb8de04c3bde301c62297127cae4d03aeb5f65b1b3b4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"e6714abbbcfe1d54c45ec38583b6367bc1667d055485de6ce77544f68acc9679","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"ccd1c15e8b2737a8cbb30d07ada8f1b6f0748afd9d43b748ec4128f7a61ce044","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879954055696384","occurred_at":"2026-08-24T13:27:00Z","signature_id":"b66537b24ffd588d399e5e7cc810bf3a188e5e0dce669b0b16956a016a18a10c","pr_number":6429,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879927736438784","occurred_at":"2026-08-24T13:26:52Z","signature_id":"6a060f99c27004cf01810556d4b6e1126f6aeee13d0cc7b3c046327b6e9bcc0d","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"25baffda5fc6f88c385cae06e401a6f4815895c798a9d1e087a18364db777535","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":53,"runs_affected":53,"failed_runs":53,"run_impact_percent":22.083333333333332,"anchor_occurrences":53,"after_last_push_count":73,"trend_counts":[0,14,39,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":387,"prior_last_seen_at":"2026-08-22T05:45:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":53}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T20:33:27Z\n Ended: 2026-08-25T20:49:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3938560-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdqmpu9g21s6r7rqhvk6panm8pnsp6m-c6e0q4y6p6o1k6m\", pod=\"router-dcc9648b5-65gvb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdqmpu9g21s6r7rqhvk6panm8pnsp6m-c6e0q4y6p6o1k6m/router-dcc9648b5-65gvb has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T20:35:28Z\n Ended: 2026-08-25T20:47:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3938560-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdqneq42nvoid2492u6ourvlt24sbkn-private-kas\", pod=\"router-64cc77977c-qft72\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdqneq42nvoid2492u6ourvlt24sbkn-private-kas/router-64cc77977c-qft72 has been in a non-ready state for longer than 5 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T20:01:01Z\n Ended: 2026-08-25T20:16:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0886784-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdq6r93avnmkufpa2dlhkdn5bhh4958-c6q7t8b4b8e6m3v\", pod=\"router-7dffdfb86b-r9dqt\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdq6r93avnmkufpa2dlhkdn5bhh4958-c6q7t8b4b8e6m3v/router-7dffdfb86b-r9dqt has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T20:01:01Z\n Ended: 2026-08-25T20:17:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0886784-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdq785qenoju9gska8l0aimh5kanj73-r9g0o6t5l6s9a3i\", pod=\"router-87ddfb859-rc8wd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdq785qenoju9gska8l0aimh5kanj73-r9g0o6t5l6s9a3i/router-87ddfb859-rc8wd has been in a non-ready state for longer than 5 minutes.","alert fired 44 time(s) (4 unknown, 40 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T19:36:34Z\n Ended: 2026-08-25T19:42:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8405376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdppr507knoluq4b6simih0av16epoc-p8e6x0p1e8y3d3q\", pod=\"cluster-network-operator-7fc64bc864-z8m58\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdppr507knoluq4b6simih0av16epoc-p8e6x0p1e8y3d3q/cluster-network-operator-7fc64bc864-z8m58 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T19:37:33Z\n Ended: 2026-08-25T19:43:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8405376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdppu4r7j3gvpcpphqjg7kspjjb2d5a-ea-listmdvtnb\", pod=\"cluster-network-operator-bc9698665-b8c6g\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdppu4r7j3gvpcpphqjg7kspjjb2d5a-ea-listmdvtnb/cluster-network-operator-bc9698665-b8c6g has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T19:37:34Z\n Ended: 2026-08-25T19:43:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8405376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdppt1p2lo3ijsep2g5q057lthm1bnq-y8l2t6b7a3x5m5p\", pod=\"ingress-operator-9dfd956c9-slrm6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdppt1p2lo3ijsep2g5q057lthm1bnq-y8l2t6b7a3x5m5p/ingress-operator-9dfd956c9-slrm6 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T19:37:34Z\n Ended: 2026-08-25T19:43:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8405376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdppu4r7j3gvpcpphqjg7kspjjb2d5a-ea-listmdvtnb\", pod=\"ingress-operator-6f9986dfc-xd7sz\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdppu4r7j3gvpcpphqjg7kspjjb2d5a-ea-listmdvtnb/ingress-operator-6f9986dfc-xd7sz has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092336210943938560","occurred_at":"2026-08-25T19:39:54Z","signature_id":"039fd788cad621fcb3beb9cbf72cb4499738c76ca4a7ab06ca2dffe150368de9","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"c8e2dae791d9eee6c2dfe89ff5eb24b232035b0f6a8f9722f2f09c4da4c46270","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"00cf1cdbbe1c25edacd565cc3dbeb30b1fc906ca4e493b4125b891e5503ffe76","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"670def82337f4e93346e218bb912d2f5102dda3d4e354be6f7a3ae52af837b74","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"f1a9a05cfde18cca1700edae4de1bd0b0a9bac2292183bc359385aab761dec79","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","occurred_at":"2026-08-25T16:56:19Z","signature_id":"e5fda378ff57d5b1940c8266f347e7ee86ada79c73f8812c6767fe80cbdefd69","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284063539793920","occurred_at":"2026-08-25T16:12:48Z","signature_id":"2d9cf619745584716b1d36a78f973479ee6f6a9974e831e0e6595f6f7c09c34e","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284042668937216","occurred_at":"2026-08-25T16:12:43Z","signature_id":"84c9f5cec5ace865d6364245a39389a7b4680b6bfdf2958173dfcfd15754e303","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284038717902848","occurred_at":"2026-08-25T16:12:42Z","signature_id":"7a5c0ec1b1bbd3d90f56d54a6d79e27df4b0c9b8be003edb69252609bc761f85","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"fa55ad2520ac87c347911e5a9a91dd9cd00f969a783efd3c191a3eabc9b22503","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"945dcc48e93f7e5479b3db04e970b41db897299b7ad75aca01031fd96f7fcf6e","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","signature_id":"d5f85864d3d15c27d9d690ef128585ac4484e8c8af2bd1de02f0ebc7ecb160b9","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","occurred_at":"2026-08-25T14:43:04Z","signature_id":"6619f8482211ef7b9931e9e6e5ce9917b4a89ba14b1e55a1266a1da1dc746d2a","pr_number":6676,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"85470de5379b7a58f4475a5818e36e604131a03a4654ffc58a012b2817701ddb","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","signature_id":"ba0d4cca986340026ad28b58a3d7254c20c5a9e8dbe174366e6c0feb810b455d","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092258099980144640","occurred_at":"2026-08-25T14:29:37Z","signature_id":"c9fd5e19ecb8a19404c13852abed9dafbdc292ec71487776299a01961be38d2a","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092254086249320448","occurred_at":"2026-08-25T14:13:40Z","signature_id":"6fc405c92e08fa1576ff900b57b0c002219d66eb9e455d1fd3d73886dc969d02","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253504889425920","occurred_at":"2026-08-25T14:11:22Z","signature_id":"2cc3c6061c407d3c74ad5325b6c2ccb6ae669b0fee5d48f4299ed1a3a6b6f5c4","pr_number":6587,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"0a3222b13ca8e18b09f294cc8bf6c58f2b714b4f68671591fc81d113df214527","pr_number":6630,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","signature_id":"7d256492ee5379d7f038bf539080c373cfbf081cb07ec0c35738f57cdb906cc2","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","signature_id":"85d3fb950aa7a81bc280ad8111f550c48065298ea6e7ee23a30155e0451f6699","pr_number":6677,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","occurred_at":"2026-08-25T13:34:16Z","signature_id":"46fe202d87700d2141f9c24fa084e86f0be02779da003c0aadb556639c6b00c1","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"142358f238029e754285919ac9ae3371cf3d7fcc3f75f1908d60371d3bae5134","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"1ae78d946fc05772983eafd321d0a8f04e93c242870b0eb8a31923e9082da049","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197879035727872","occurred_at":"2026-08-25T10:30:20Z","signature_id":"9f099708799bf3e2ac428bc43b19f24600de3b38fc32adfc9aa785b8a28ba4cc","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"9323f34e5dc7624ff70491391f71624b8fccbc2b4b0fdf3ebe01e36965be3373","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"1399061bb15ba2afe3a3d49ef0c1c03b496dabeb92275fbed52214cd772f172d","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"ab1ad628fd91e461795e6817e8295d83b1b66efac12206305c9fb1793f4943a9","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"9a2d5923b47e722ec14a03cd82e80a5847eb6af6c56086cf441bc75f54feb7ce","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"783b9a11bba8d6f9b48cde87f4a3968184ce14d623eaa09a0d5aae21b1c10281","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"6791742669eeec9806dabaab1695d3466c0bad5f4d644012bab5a19c0d345acd","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"1418018ee02cdd89dd6ac2c5c6e1378a62f42c5f58ff6ea19b95f13d3e928b4a","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"e906e7ae8a41e3b71c67202b1c3d92e471ada05f8b75a59665632b01eb944b26","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146681486249984","occurred_at":"2026-08-25T07:06:53Z","signature_id":"85a4cfecb0adb1b2747bda34b1b1285829950a4778f69e05021f9596834b1154","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"7151c69efd10f0212e88001c4fa93856449529055b2b769b7fc63617db078de2","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","signature_id":"9912e6cb8637cbc60f5d14947b0d4860ec095a6f064bb2b7e514dd1160ef1a11","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","signature_id":"939ee2aa813aeb34b5caee82729b4311d14f106923472e3fd6fed1c4ae71c667","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"04107c06cdc1c97e2051bd30a01f9feaff1579d56b011fda20b5f0cc7435b337","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","occurred_at":"2026-08-25T00:21:02Z","signature_id":"51d2fea77be7cecb840acdb29e21e26279922de433103aefa5ec401ed53ade1f","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","signature_id":"2f1d9b2b8857e75d809a23da90c2e32bd0a70345c25f6862fc7292ac5c796db3","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","signature_id":"f8ca631bf4ba3de6a823751843028c53bf8f63552f781736fc0f06124e42925e","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"762c894801e83a205ad61ad7c1a1171fcbbd7db65d82ac38613bccfd72e8ae0d","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"5a62ef9a7062e35c1e10c6c46839368f20045e53d18429e0c93ad1b8118f4ed4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"05c28635da15b6abb7fa52304cc82cedf7070115a67070cb7058c7bc17ceed55","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","signature_id":"d9b2b2f48eba30a813d1493c2cf9d779d74129e8e68690c29d48df3b77d5680a","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"84936854ba2393da3d6d4a90cbe465eac2bfa99410eb5ae262e1869d118d0d44","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"d6263c32bd22604b284f6288625ad3fcd30c678787956b94fe5bf7e9a8e0f6cc","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"ddb17f85d08dfa0f0d60af8e146b060c3f080bb3cb873c1fd946241da0c96c74","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091920407983558656","occurred_at":"2026-08-24T16:06:36Z","signature_id":"c52899aff7bf73e0f12d284cdf7344825014b3fd387fb2f270a8871271bc7acb","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"32b177871d4669aa26bebdb303a13d4eeafa32be58fe0c29eecd2b7355b315ff","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"d1d5a7d0ff60a9161bf6b8e8b3cd897227641812139ecab6ded5ff874a074bf7","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"41f9fa028fb65313589b4f31abf40b7c37b7c57dd66b9ac548b1148f02514739","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"d55af07fa2830b9ad7b195c673e4e375ec2e39f3e870f6e6e4139ca2af8abfa0","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":49,"runs_affected":49,"failed_runs":49,"run_impact_percent":20.416666666666668,"anchor_occurrences":49,"after_last_push_count":0,"trend_counts":[0,35,14,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":49,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":49}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0006c4270\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[08:07:56.002]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:43.2162872Z);\\nlet _endTime = datetime(2026-08-25T08:07:43.2162878Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:07:55.9842316Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f\\\",\\r\\n \\\"activityId\\\": \\\"93240ce7-344d-4b28-a6d5-bc7c0d41eeab\\\",\\r\\n \\\"subActivityId\\\": \\\"daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2d5a9b37-61e4-4d07-80a4-b16788917aef\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f ARID=93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.Http.CallContext/93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.ExecuteQuery/2d5a9b37-61e4-4d07-80a4-b16788917aef \\u003e DN.FE.ExecuteQuery/daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[08:07:56.054]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:43.2162872Z);\\nlet _endTime = datetime(2026-08-25T08:07:43.2162878Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:07:55.9842316Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f\\\",\\r\\n \\\"activityId\\\": \\\"93240ce7-344d-4b28-a6d5-bc7c0d41eeab\\\",\\r\\n \\\"subActivityId\\\": \\\"daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2d5a9b37-61e4-4d07-80a4-b16788917aef\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f ARID=93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.Http.CallContext/93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.ExecuteQuery/2d5a9b37-61e4-4d07-80a4-b16788917aef \\u003e DN.FE.ExecuteQuery/daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[08:08:04.153]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:56.1417247Z);\\nlet _endTime = datetime(2026-08-25T08:07:56.1417253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:04.1346416Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42\\\",\\r\\n \\\"activityId\\\": \\\"5170d421-679a-4a48-b29b-9a996c74dde4\\\",\\r\\n \\\"subActivityId\\\": \\\"d37306ab-e852-439a-bb92-4f7c817288f6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"36329843-81dc-4dc4-aa28-c56ee7342c50\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42 ARID=5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.Http.CallContext/5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.ExecuteQuery/36329843-81dc-4dc4-aa28-c56ee7342c50 \\u003e DN.FE.ExecuteQuery/d37306ab-e852-439a-bb92-4f7c817288f6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[08:08:04.206]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:56.1417247Z);\\nlet _endTime = datetime(2026-08-25T08:07:56.1417253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:04.1346416Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42\\\",\\r\\n \\\"activityId\\\": \\\"5170d421-679a-4a48-b29b-9a996c74dde4\\\",\\r\\n \\\"subActivityId\\\": \\\"d37306ab-e852-439a-bb92-4f7c817288f6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"36329843-81dc-4dc4-aa28-c56ee7342c50\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42 ARID=5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.Http.CallContext/5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.ExecuteQuery/36329843-81dc-4dc4-aa28-c56ee7342c50 \\u003e DN.FE.ExecuteQuery/d37306ab-e852-439a-bb92-4f7c817288f6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[08:08:15.998]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:08:07.6782331Z);\\nlet _endTime = datetime(2026-08-25T08:08:07.6782337Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:15.9798710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b\\\",\\r\\n \\\"activityId\\\": \\\"c0534e85-0700-4932-b07d-9b9e7af63995\\\",\\r\\n \\\"subActivityId\\\": \\\"abf8d736-f9c7-4683-a287-d5c9f53d5b71\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"54a6cff9-747f-4ac5-8f21-5804ee2d36e8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b ARID=c0534e85-0700-4932-b07d-9b9e7af63995 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/54a6cff9-747f-4ac5-8f21-5804ee2d36e8 \\u003e DN.FE.ExecuteQuery/abf8d736-f9c7-4683-a287-d5c9f53d5b71)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[08:08:16.102]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:08:07.6782331Z);\\nlet _endTime = datetime(2026-08-25T08:08:07.6782337Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:15.9798710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b\\\",\\r\\n \\\"activityId\\\": \\\"c0534e85-0700-4932-b07d-9b9e7af63995\\\",\\r\\n \\\"subActivityId\\\": \\\"abf8d736-f9c7-4683-a287-d5c9f53d5b71\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"54a6cff9-747f-4ac5-8f21-5804ee2d36e8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b ARID=c0534e85-0700-4932-b07d-9b9e7af63995 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/54a6cff9-747f-4ac5-8f21-5804ee2d36e8 \\u003e DN.FE.ExecuteQuery/abf8d736-f9c7-4683-a287-d5c9f53d5b71)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0006c4270\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[08:07:56.002]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:43.2162872Z);\\nlet _endTime = datetime(2026-08-25T08:07:43.2162878Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:07:55.9842316Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f\\\",\\r\\n \\\"activityId\\\": \\\"93240ce7-344d-4b28-a6d5-bc7c0d41eeab\\\",\\r\\n \\\"subActivityId\\\": \\\"daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2d5a9b37-61e4-4d07-80a4-b16788917aef\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f ARID=93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.Http.CallContext/93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.ExecuteQuery/2d5a9b37-61e4-4d07-80a4-b16788917aef \\u003e DN.FE.ExecuteQuery/daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[08:07:56.054]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:43.2162872Z);\\nlet _endTime = datetime(2026-08-25T08:07:43.2162878Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:07:55.9842316Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f\\\",\\r\\n \\\"activityId\\\": \\\"93240ce7-344d-4b28-a6d5-bc7c0d41eeab\\\",\\r\\n \\\"subActivityId\\\": \\\"daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2d5a9b37-61e4-4d07-80a4-b16788917aef\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f ARID=93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.Http.CallContext/93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.ExecuteQuery/2d5a9b37-61e4-4d07-80a4-b16788917aef \\u003e DN.FE.ExecuteQuery/daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[08:08:04.153]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:56.1417247Z);\\nlet _endTime = datetime(2026-08-25T08:07:56.1417253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:04.1346416Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42\\\",\\r\\n \\\"activityId\\\": \\\"5170d421-679a-4a48-b29b-9a996c74dde4\\\",\\r\\n \\\"subActivityId\\\": \\\"d37306ab-e852-439a-bb92-4f7c817288f6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"36329843-81dc-4dc4-aa28-c56ee7342c50\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42 ARID=5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.Http.CallContext/5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.ExecuteQuery/36329843-81dc-4dc4-aa28-c56ee7342c50 \\u003e DN.FE.ExecuteQuery/d37306ab-e852-439a-bb92-4f7c817288f6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[08:08:04.206]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:56.1417247Z);\\nlet _endTime = datetime(2026-08-25T08:07:56.1417253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:04.1346416Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42\\\",\\r\\n \\\"activityId\\\": \\\"5170d421-679a-4a48-b29b-9a996c74dde4\\\",\\r\\n \\\"subActivityId\\\": \\\"d37306ab-e852-439a-bb92-4f7c817288f6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"36329843-81dc-4dc4-aa28-c56ee7342c50\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42 ARID=5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.Http.CallContext/5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.ExecuteQuery/36329843-81dc-4dc4-aa28-c56ee7342c50 \\u003e DN.FE.ExecuteQuery/d37306ab-e852-439a-bb92-4f7c817288f6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[08:08:15.998]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:08:07.6782331Z);\\nlet _endTime = datetime(2026-08-25T08:08:07.6782337Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:15.9798710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b\\\",\\r\\n \\\"activityId\\\": \\\"c0534e85-0700-4932-b07d-9b9e7af63995\\\",\\r\\n \\\"subActivityId\\\": \\\"abf8d736-f9c7-4683-a287-d5c9f53d5b71\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"54a6cff9-747f-4ac5-8f21-5804ee2d36e8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b ARID=c0534e85-0700-4932-b07d-9b9e7af63995 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/54a6cff9-747f-4ac5-8f21-5804ee2d36e8 \\u003e DN.FE.ExecuteQuery/abf8d736-f9c7-4683-a287-d5c9f53d5b71)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[08:08:16.102]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:08:07.6782331Z);\\nlet _endTime = datetime(2026-08-25T08:08:07.6782337Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:15.9798710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b\\\",\\r\\n \\\"activityId\\\": \\\"c0534e85-0700-4932-b07d-9b9e7af63995\\\",\\r\\n \\\"subActivityId\\\": \\\"abf8d736-f9c7-4683-a287-d5c9f53d5b71\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"54a6cff9-747f-4ac5-8f21-5804ee2d36e8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b ARID=c0534e85-0700-4932-b07d-9b9e7af63995 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/54a6cff9-747f-4ac5-8f21-5804ee2d36e8 \\u003e DN.FE.ExecuteQuery/abf8d736-f9c7-4683-a287-d5c9f53d5b71)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000812850\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[07:36:48.911]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:45.8465920Z);\\nlet _endTime = datetime(2026-08-25T07:36:45.8465927Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:48.8924657Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1\\\",\\r\\n \\\"activityId\\\": \\\"2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672\\\",\\r\\n \\\"subActivityId\\\": \\\"403b7745-bbff-4c9f-929d-476fbd66118e\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e88d573b-aeb8-4a17-9fca-c87f04c6cdd9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1 ARID=2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e88d573b-aeb8-4a17-9fca-c87f04c6cdd9 \\u003e DN.FE.ExecuteQuery/403b7745-bbff-4c9f-929d-476fbd66118e)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:36:48.962]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:45.8465920Z);\\nlet _endTime = datetime(2026-08-25T07:36:45.8465927Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:48.8924657Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1\\\",\\r\\n \\\"activityId\\\": \\\"2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672\\\",\\r\\n \\\"subActivityId\\\": \\\"403b7745-bbff-4c9f-929d-476fbd66118e\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e88d573b-aeb8-4a17-9fca-c87f04c6cdd9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1 ARID=2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e88d573b-aeb8-4a17-9fca-c87f04c6cdd9 \\u003e DN.FE.ExecuteQuery/403b7745-bbff-4c9f-929d-476fbd66118e)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[07:36:51.855]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:49.0279421Z);\\nlet _endTime = datetime(2026-08-25T07:36:49.0279424Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:51.8360877Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b\\\",\\r\\n \\\"activityId\\\": \\\"f4d03f54-2298-4d29-b8e3-9d984c0bd2b9\\\",\\r\\n \\\"subActivityId\\\": \\\"60e487db-9b15-48f2-a301-b66685f17ff4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4dc8f887-d2c7-4ef0-ad29-017a6723ebe4\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b ARID=f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.Http.CallContext/f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.ExecuteQuery/4dc8f887-d2c7-4ef0-ad29-017a6723ebe4 \\u003e DN.FE.ExecuteQuery/60e487db-9b15-48f2-a301-b66685f17ff4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:36:51.908]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:49.0279421Z);\\nlet _endTime = datetime(2026-08-25T07:36:49.0279424Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:51.8360877Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b\\\",\\r\\n \\\"activityId\\\": \\\"f4d03f54-2298-4d29-b8e3-9d984c0bd2b9\\\",\\r\\n \\\"subActivityId\\\": \\\"60e487db-9b15-48f2-a301-b66685f17ff4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4dc8f887-d2c7-4ef0-ad29-017a6723ebe4\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b ARID=f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.Http.CallContext/f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.ExecuteQuery/4dc8f887-d2c7-4ef0-ad29-017a6723ebe4 \\u003e DN.FE.ExecuteQuery/60e487db-9b15-48f2-a301-b66685f17ff4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[07:36:58.311]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:55.2926733Z);\\nlet _endTime = datetime(2026-08-25T07:36:55.2926740Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:58.2926720Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958\\\",\\r\\n \\\"activityId\\\": \\\"5168b874-e371-442d-96d4-634b404e01b7\\\",\\r\\n \\\"subActivityId\\\": \\\"a4dec94c-d4be-425e-8443-3e3dba0c4a46\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"33d6b2a1-bd08-472a-87a5-98f6eafd09fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958 ARID=5168b874-e371-442d-96d4-634b404e01b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/33d6b2a1-bd08-472a-87a5-98f6eafd09fb \\u003e DN.FE.ExecuteQuery/a4dec94c-d4be-425e-8443-3e3dba0c4a46)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:36:58.405]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:55.2926733Z);\\nlet _endTime = datetime(2026-08-25T07:36:55.2926740Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:58.2926720Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958\\\",\\r\\n \\\"activityId\\\": \\\"5168b874-e371-442d-96d4-634b404e01b7\\\",\\r\\n \\\"subActivityId\\\": \\\"a4dec94c-d4be-425e-8443-3e3dba0c4a46\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"33d6b2a1-bd08-472a-87a5-98f6eafd09fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958 ARID=5168b874-e371-442d-96d4-634b404e01b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/33d6b2a1-bd08-472a-87a5-98f6eafd09fb \\u003e DN.FE.ExecuteQuery/a4dec94c-d4be-425e-8443-3e3dba0c4a46)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000812850\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[07:36:48.911]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:45.8465920Z);\\nlet _endTime = datetime(2026-08-25T07:36:45.8465927Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:48.8924657Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1\\\",\\r\\n \\\"activityId\\\": \\\"2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672\\\",\\r\\n \\\"subActivityId\\\": \\\"403b7745-bbff-4c9f-929d-476fbd66118e\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e88d573b-aeb8-4a17-9fca-c87f04c6cdd9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1 ARID=2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e88d573b-aeb8-4a17-9fca-c87f04c6cdd9 \\u003e DN.FE.ExecuteQuery/403b7745-bbff-4c9f-929d-476fbd66118e)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:36:48.962]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:45.8465920Z);\\nlet _endTime = datetime(2026-08-25T07:36:45.8465927Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:48.8924657Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1\\\",\\r\\n \\\"activityId\\\": \\\"2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672\\\",\\r\\n \\\"subActivityId\\\": \\\"403b7745-bbff-4c9f-929d-476fbd66118e\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e88d573b-aeb8-4a17-9fca-c87f04c6cdd9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1 ARID=2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e88d573b-aeb8-4a17-9fca-c87f04c6cdd9 \\u003e DN.FE.ExecuteQuery/403b7745-bbff-4c9f-929d-476fbd66118e)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[07:36:51.855]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:49.0279421Z);\\nlet _endTime = datetime(2026-08-25T07:36:49.0279424Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:51.8360877Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b\\\",\\r\\n \\\"activityId\\\": \\\"f4d03f54-2298-4d29-b8e3-9d984c0bd2b9\\\",\\r\\n \\\"subActivityId\\\": \\\"60e487db-9b15-48f2-a301-b66685f17ff4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4dc8f887-d2c7-4ef0-ad29-017a6723ebe4\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b ARID=f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.Http.CallContext/f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.ExecuteQuery/4dc8f887-d2c7-4ef0-ad29-017a6723ebe4 \\u003e DN.FE.ExecuteQuery/60e487db-9b15-48f2-a301-b66685f17ff4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:36:51.908]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:49.0279421Z);\\nlet _endTime = datetime(2026-08-25T07:36:49.0279424Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:51.8360877Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b\\\",\\r\\n \\\"activityId\\\": \\\"f4d03f54-2298-4d29-b8e3-9d984c0bd2b9\\\",\\r\\n \\\"subActivityId\\\": \\\"60e487db-9b15-48f2-a301-b66685f17ff4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4dc8f887-d2c7-4ef0-ad29-017a6723ebe4\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b ARID=f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.Http.CallContext/f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.ExecuteQuery/4dc8f887-d2c7-4ef0-ad29-017a6723ebe4 \\u003e DN.FE.ExecuteQuery/60e487db-9b15-48f2-a301-b66685f17ff4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[07:36:58.311]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:55.2926733Z);\\nlet _endTime = datetime(2026-08-25T07:36:55.2926740Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:58.2926720Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958\\\",\\r\\n \\\"activityId\\\": \\\"5168b874-e371-442d-96d4-634b404e01b7\\\",\\r\\n \\\"subActivityId\\\": \\\"a4dec94c-d4be-425e-8443-3e3dba0c4a46\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"33d6b2a1-bd08-472a-87a5-98f6eafd09fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958 ARID=5168b874-e371-442d-96d4-634b404e01b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/33d6b2a1-bd08-472a-87a5-98f6eafd09fb \\u003e DN.FE.ExecuteQuery/a4dec94c-d4be-425e-8443-3e3dba0c4a46)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:36:58.405]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:55.2926733Z);\\nlet _endTime = datetime(2026-08-25T07:36:55.2926740Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:58.2926720Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958\\\",\\r\\n \\\"activityId\\\": \\\"5168b874-e371-442d-96d4-634b404e01b7\\\",\\r\\n \\\"subActivityId\\\": \\\"a4dec94c-d4be-425e-8443-3e3dba0c4a46\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"33d6b2a1-bd08-472a-87a5-98f6eafd09fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958 ARID=5168b874-e371-442d-96d4-634b404e01b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/33d6b2a1-bd08-472a-87a5-98f6eafd09fb \\u003e DN.FE.ExecuteQuery/a4dec94c-d4be-425e-8443-3e3dba0c4a46)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000a22250\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[06:58:42.889]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:39.6001491Z);\\nlet _endTime = datetime(2026-08-25T06:58:39.6001496Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:42.8701809Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c\\\",\\r\\n \\\"activityId\\\": \\\"50929d0b-6732-4e31-a429-a529a01fac9c\\\",\\r\\n \\\"subActivityId\\\": \\\"29a9a3a1-3134-4073-9f9f-167f7be9ed6d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9e96b72c-b3c0-4907-ba16-c5f840899e89\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c ARID=50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.Http.CallContext/50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.ExecuteQuery/9e96b72c-b3c0-4907-ba16-c5f840899e89 \\u003e DN.FE.ExecuteQuery/29a9a3a1-3134-4073-9f9f-167f7be9ed6d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:58:42.938]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:39.6001491Z);\\nlet _endTime = datetime(2026-08-25T06:58:39.6001496Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:42.8701809Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c\\\",\\r\\n \\\"activityId\\\": \\\"50929d0b-6732-4e31-a429-a529a01fac9c\\\",\\r\\n \\\"subActivityId\\\": \\\"29a9a3a1-3134-4073-9f9f-167f7be9ed6d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9e96b72c-b3c0-4907-ba16-c5f840899e89\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c ARID=50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.Http.CallContext/50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.ExecuteQuery/9e96b72c-b3c0-4907-ba16-c5f840899e89 \\u003e DN.FE.ExecuteQuery/29a9a3a1-3134-4073-9f9f-167f7be9ed6d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[06:58:46.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:43.0033907Z);\\nlet _endTime = datetime(2026-08-25T06:58:43.0033913Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:46.4202164Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2b381053-8eff-40bc-996d-50543376f61a\\\",\\r\\n \\\"activityId\\\": \\\"4fca92f9-d3e2-4fa6-81c9-a9fc8513a713\\\",\\r\\n \\\"subActivityId\\\": \\\"3821d24b-5b2c-40a4-bcae-6560c92f6b97\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"48e1149d-7cf7-4df9-9a59-15c62ac82ce9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2b381053-8eff-40bc-996d-50543376f61a ARID=4fca92f9-d3e2-4fa6-81c9-a9fc8513a713 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/48e1149d-7cf7-4df9-9a59-15c62ac82ce9 \\u003e DN.FE.ExecuteQuery/3821d24b-5b2c-40a4-bcae-6560c92f6b97)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:58:46.487]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:43.0033907Z);\\nlet _endTime = datetime(2026-08-25T06:58:43.0033913Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:46.4202164Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2b381053-8eff-40bc-996d-50543376f61a\\\",\\r\\n \\\"activityId\\\": \\\"4fca92f9-d3e2-4fa6-81c9-a9fc8513a713\\\",\\r\\n \\\"subActivityId\\\": \\\"3821d24b-5b2c-40a4-bcae-6560c92f6b97\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"48e1149d-7cf7-4df9-9a59-15c62ac82ce9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2b381053-8eff-40bc-996d-50543376f61a ARID=4fca92f9-d3e2-4fa6-81c9-a9fc8513a713 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/48e1149d-7cf7-4df9-9a59-15c62ac82ce9 \\u003e DN.FE.ExecuteQuery/3821d24b-5b2c-40a4-bcae-6560c92f6b97)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[06:58:53.091]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:49.9195659Z);\\nlet _endTime = datetime(2026-08-25T06:58:49.9195666Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:53.0733729Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf\\\",\\r\\n \\\"activityId\\\": \\\"8f036878-c750-44d8-94c2-d0ffa48c5023\\\",\\r\\n \\\"subActivityId\\\": \\\"f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ecd293b9-988c-49f9-988b-2c0d898fd0bb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf ARID=8f036878-c750-44d8-94c2-d0ffa48c5023 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecd293b9-988c-49f9-988b-2c0d898fd0bb \\u003e DN.FE.ExecuteQuery/f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:58:53.185]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:49.9195659Z);\\nlet _endTime = datetime(2026-08-25T06:58:49.9195666Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:53.0733729Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf\\\",\\r\\n \\\"activityId\\\": \\\"8f036878-c750-44d8-94c2-d0ffa48c5023\\\",\\r\\n \\\"subActivityId\\\": \\\"f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ecd293b9-988c-49f9-988b-2c0d898fd0bb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf ARID=8f036878-c750-44d8-94c2-d0ffa48c5023 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecd293b9-988c-49f9-988b-2c0d898fd0bb \\u003e DN.FE.ExecuteQuery/f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000a22250\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[06:58:42.889]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:39.6001491Z);\\nlet _endTime = datetime(2026-08-25T06:58:39.6001496Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:42.8701809Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c\\\",\\r\\n \\\"activityId\\\": \\\"50929d0b-6732-4e31-a429-a529a01fac9c\\\",\\r\\n \\\"subActivityId\\\": \\\"29a9a3a1-3134-4073-9f9f-167f7be9ed6d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9e96b72c-b3c0-4907-ba16-c5f840899e89\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c ARID=50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.Http.CallContext/50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.ExecuteQuery/9e96b72c-b3c0-4907-ba16-c5f840899e89 \\u003e DN.FE.ExecuteQuery/29a9a3a1-3134-4073-9f9f-167f7be9ed6d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:58:42.938]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:39.6001491Z);\\nlet _endTime = datetime(2026-08-25T06:58:39.6001496Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:42.8701809Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c\\\",\\r\\n \\\"activityId\\\": \\\"50929d0b-6732-4e31-a429-a529a01fac9c\\\",\\r\\n \\\"subActivityId\\\": \\\"29a9a3a1-3134-4073-9f9f-167f7be9ed6d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9e96b72c-b3c0-4907-ba16-c5f840899e89\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c ARID=50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.Http.CallContext/50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.ExecuteQuery/9e96b72c-b3c0-4907-ba16-c5f840899e89 \\u003e DN.FE.ExecuteQuery/29a9a3a1-3134-4073-9f9f-167f7be9ed6d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[06:58:46.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:43.0033907Z);\\nlet _endTime = datetime(2026-08-25T06:58:43.0033913Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:46.4202164Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2b381053-8eff-40bc-996d-50543376f61a\\\",\\r\\n \\\"activityId\\\": \\\"4fca92f9-d3e2-4fa6-81c9-a9fc8513a713\\\",\\r\\n \\\"subActivityId\\\": \\\"3821d24b-5b2c-40a4-bcae-6560c92f6b97\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"48e1149d-7cf7-4df9-9a59-15c62ac82ce9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2b381053-8eff-40bc-996d-50543376f61a ARID=4fca92f9-d3e2-4fa6-81c9-a9fc8513a713 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/48e1149d-7cf7-4df9-9a59-15c62ac82ce9 \\u003e DN.FE.ExecuteQuery/3821d24b-5b2c-40a4-bcae-6560c92f6b97)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:58:46.487]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:43.0033907Z);\\nlet _endTime = datetime(2026-08-25T06:58:43.0033913Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:46.4202164Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2b381053-8eff-40bc-996d-50543376f61a\\\",\\r\\n \\\"activityId\\\": \\\"4fca92f9-d3e2-4fa6-81c9-a9fc8513a713\\\",\\r\\n \\\"subActivityId\\\": \\\"3821d24b-5b2c-40a4-bcae-6560c92f6b97\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"48e1149d-7cf7-4df9-9a59-15c62ac82ce9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2b381053-8eff-40bc-996d-50543376f61a ARID=4fca92f9-d3e2-4fa6-81c9-a9fc8513a713 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/48e1149d-7cf7-4df9-9a59-15c62ac82ce9 \\u003e DN.FE.ExecuteQuery/3821d24b-5b2c-40a4-bcae-6560c92f6b97)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[06:58:53.091]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:49.9195659Z);\\nlet _endTime = datetime(2026-08-25T06:58:49.9195666Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:53.0733729Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf\\\",\\r\\n \\\"activityId\\\": \\\"8f036878-c750-44d8-94c2-d0ffa48c5023\\\",\\r\\n \\\"subActivityId\\\": \\\"f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ecd293b9-988c-49f9-988b-2c0d898fd0bb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf ARID=8f036878-c750-44d8-94c2-d0ffa48c5023 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecd293b9-988c-49f9-988b-2c0d898fd0bb \\u003e DN.FE.ExecuteQuery/f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:58:53.185]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:49.9195659Z);\\nlet _endTime = datetime(2026-08-25T06:58:49.9195666Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:53.0733729Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf\\\",\\r\\n \\\"activityId\\\": \\\"8f036878-c750-44d8-94c2-d0ffa48c5023\\\",\\r\\n \\\"subActivityId\\\": \\\"f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ecd293b9-988c-49f9-988b-2c0d898fd0bb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf ARID=8f036878-c750-44d8-94c2-d0ffa48c5023 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecd293b9-988c-49f9-988b-2c0d898fd0bb \\u003e DN.FE.ExecuteQuery/f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092142908357480448","occurred_at":"2026-08-25T06:51:54Z","signature_id":"02210cdb0477006b4ad2f63d27bbb22b1fec432dfc421b3f4927ea8331e71bd5","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"ae0283cab4bf11ad3998038231a262c024b0853fa925e9504e4d72e900d671b8","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092131475158732800","occurred_at":"2026-08-25T06:06:28Z","signature_id":"8f69b74b7621ff0b0b8ce3121e6614ce9d3311a7090b53f7507107351ee1dd64","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","signature_id":"09aaf44a85636c53922c8c55c3f077fa7ba9e0be3e43723e44ff0ee719184329","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092122658786250752","occurred_at":"2026-08-25T05:31:26Z","signature_id":"0a451d1043de67f733809747c484d6352481472e46de0c7c6b4a4bb3a65e52d5","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092101471658053632","occurred_at":"2026-08-25T04:07:14Z","signature_id":"f34a1ababae19d28287c5f9ad6fadbb834adfab66885b9336868ab7e5fe3c4e0","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","signature_id":"ec3713fc3fd2cd0078c515157ab16cbcdeafc43ac2e38c3ebeac3729eec40f03","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092091165758197760","occurred_at":"2026-08-25T03:26:17Z","signature_id":"0656eacbaaee9c8627f179735f527bbfa1a360407e229ef55d81deb432be6cbc","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092089899510403072","occurred_at":"2026-08-25T03:21:15Z","signature_id":"b479792efcddc2bca8690264151fa877f3ce0d483876238bd57c9976805c7430","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071627658366976","occurred_at":"2026-08-25T02:08:39Z","signature_id":"c9f1b4828c5cc783bc0db2584b06c52c8e5851d35045a6f094b2bdb76e6aa487","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071623086575616","occurred_at":"2026-08-25T02:08:38Z","signature_id":"28297427d9d2bd8653dc649081b99f9573b907029f55909add770c57baa41099","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092066535932694528","occurred_at":"2026-08-25T01:48:25Z","signature_id":"002c1d2e05187089a12c62fceba5d319b6b2b88aa8f59eba28a6c84b20651f56","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092066407356305408","occurred_at":"2026-08-25T01:47:54Z","signature_id":"805bce7e0aab4147685c771f0375ca865818fb3186f1e427a03da21aa4561fc2","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092041100716937216","occurred_at":"2026-08-25T00:07:21Z","signature_id":"938c8e2babfc879ca08c00539016e48f5863191d883d5d8b5e82fc0bb437190d","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","signature_id":"93cd087eb0fc0ac2b0c79ba3bc6c6181f71cba9db1ac3eaefd4c1d8152fbb877","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092031971118026752","occurred_at":"2026-08-24T23:30:58Z","signature_id":"176c7f8e0e9a6fe4e5a12f9055659eede136ab94bf2bc621167d758c6dc1828c","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092030482601480192","occurred_at":"2026-08-24T23:25:09Z","signature_id":"a4d769910ea57a048ae6030e8851eaac93f508a9d62968944253d616d8fe88ed","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092025885698822144","occurred_at":"2026-08-24T23:06:53Z","signature_id":"9eb8d09c21becab11e4df2e2486fbadfcfe682a12484ff5957cc08b54e0cd902","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","signature_id":"a99ad7aa848b9f85d32a71fb35dfd49c3d8270fdc830d4478b6af42e4702a425","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","signature_id":"0d5221b00aeb7bfdca1da1581a9d7578e9d19b098899c13f87a46837f2394f22","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092014812023230464","occurred_at":"2026-08-24T22:22:53Z","signature_id":"c5fd89f1c41b331bfa0c3c5ff918b5ee94282a73960beb6c8680dfabd0d92aba","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092003732647579648","occurred_at":"2026-08-24T21:38:47Z","signature_id":"c3920cca5c3b22e9ea0a8df63d7fd82dff370a2c49fd2a56d2b8612e6611e8cd","pr_number":6088,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091996785936437248","occurred_at":"2026-08-24T21:11:15Z","signature_id":"c79de5565353120aa973caf9da2e4e0043d28fd25ac56a7cd204827ac9e612e5","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091995273302642688","occurred_at":"2026-08-24T21:05:15Z","signature_id":"b19e1030a263db434fc5938127062d3add16c8d3a56319f83b1663785d30bba9","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"98f2c85bfff864b394f0a6540743e0876e4af34e9f265ec3b983e45e8ef64a57","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"fa1482e0c10fe89a03896ac5f14bc5ff11cbc2b5c024fd993535afa6403d322b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985911028912128","occurred_at":"2026-08-24T20:28:03Z","signature_id":"43910364f48f66523f1f9d08a417492faf101b0ab2cb444442ab1a553439c6b5","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985362346840064","occurred_at":"2026-08-24T20:25:41Z","signature_id":"176cc4504c2f048683d4e9fd8346ddda26590ba46f5fcff44ce3756a85f5565e","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"913731872ab076d2435270c0fedcc39b2d88a5ff36047161ca4eabe2b648b67d","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091978207036182528","occurred_at":"2026-08-24T19:57:24Z","signature_id":"f663b43ed0edc7e9f34c5b7d6058cf02871c4b2cbf7c46200eb2800bb395c1fc","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977503907254272","occurred_at":"2026-08-24T19:54:38Z","signature_id":"0b9a16666408f3f71c77e5a20e559de4e428d06efaede29056c5fdec679a0549","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977495384428544","occurred_at":"2026-08-24T19:54:36Z","signature_id":"07fcf787e9667bb548a1ae4e2aa892f82683b85aa913615ccc3c55f328e44f13","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977485536202752","occurred_at":"2026-08-24T19:54:34Z","signature_id":"edf4f7b0d5436e65624e47252a3c9819a947e53aeb06b94097e54f965d358ffa","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","signature_id":"69234d1190be6bae030c9f0be83f507930b8aba09ce9643964f39fe12af78ce2","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963250454302720","occurred_at":"2026-08-24T18:58:00Z","signature_id":"1bd7b81142bf5c01b0963c412981b707f78cb8184adf320516339e4aa3a3ce9f","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963205222928384","occurred_at":"2026-08-24T18:57:49Z","signature_id":"9bfada2b49b3fc5245efb1ec3e602fd604d8132a904f08045a044c170e893680","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"c742492599a708bf77717b7e4360828e91602b79b9e3f94c759bffe66f0abf90","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091960563839012864","occurred_at":"2026-08-24T18:47:19Z","signature_id":"ec35c9667d01d0fca7a963152324f60489cc7cd6ddc292c3798abbfb8aa51092","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091956227226472448","occurred_at":"2026-08-24T18:30:02Z","signature_id":"335052cc293f1dcfc9068c799f022fd529e07859d0cea778ae29ff1ed7adcc44","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091950955841982464","occurred_at":"2026-08-24T18:09:09Z","signature_id":"c0cebe2544e94f85956b2aa4e6cb37d39972915968a1824b87954f1d8f2b0278","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091948175697907712","occurred_at":"2026-08-24T17:58:06Z","signature_id":"2cfd9e8dd04e05dbf25d7770f4e1349a8d7b20bfb77b1b160b57373b94af3940","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091942092610736128","occurred_at":"2026-08-24T17:33:55Z","signature_id":"41ffa53c8fc1675a8e8f11ea4fff86270a85dca8d99ed458236b7319ea0aa4d4","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"dec2f42fbc616357b928df0a2e19436696b2e782470cc0447de0e33d58d63917","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932257060130816","occurred_at":"2026-08-24T16:54:39Z","signature_id":"cff96711d2e8bb5b9f48911a18c760f7b843abef0ee0b1364775c2b0b5f58a49","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932359686361088","occurred_at":"2026-08-24T16:54:34Z","signature_id":"5fa5e4ca1b4d0f869a74c660939188cb0fcaacbc16b74da34e727232c29abbff","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"6ff77bfa149b82860b0429a7dfe5ab2b71a58982ddc24dc6fbf5996e7d1dca8d","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"1ae3c1ffd864572007c734910a8d27ab7b2bea356002ed663c85c3e98b525248","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"9865c7135fd03ceb69a76994157b6614c9b64d8eebb69b3b00180111a55f75a4","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091926769790619648","occurred_at":"2026-08-24T16:21:04Z","signature_id":"40f2371fc4b699bfbeed8a52087620105728edb5da70261d5e9bd7df99b13e6a","pr_number":6639,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":46,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":46,"after_last_push_count":0,"trend_counts":[0,0,46,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f12f40\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-f5xqwdq9mzqv\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b936c26d-3470-4f47-84bd-2b15b3591b1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b936c26d-3470-4f47-84bd-2b15b3591b1e\",\n \"name\": \"b936c26d-3470-4f47-84bd-2b15b3591b1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:49:19.625710695Z\",\n \"endTime\": \"2026-08-25T20:08:26.92099949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f12f40\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-f5xqwdq9mzqv\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b936c26d-3470-4f47-84bd-2b15b3591b1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b936c26d-3470-4f47-84bd-2b15b3591b1e\",\n \"name\": \"b936c26d-3470-4f47-84bd-2b15b3591b1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:49:19.625710695Z\",\n \"endTime\": \"2026-08-25T20:08:26.92099949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e28d80\u003e: \n failed to create HCP cluster np-dg-minor-zpsckm: failed waiting for cluster=\"np-dg-minor-zpsckm\" in resourcegroup=\"rg-np-dg-minor-zpsckm-7j842f42696c\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c906de65-e3a2-4879-9236-7b176a8d1899\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c906de65-e3a2-4879-9236-7b176a8d1899\",\n \"name\": \"c906de65-e3a2-4879-9236-7b176a8d1899\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:38.123542143Z\",\n \"endTime\": \"2026-08-25T20:07:46.9158638Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e28d80\u003e: \n failed to create HCP cluster np-dg-minor-zpsckm: failed waiting for cluster=\"np-dg-minor-zpsckm\" in resourcegroup=\"rg-np-dg-minor-zpsckm-7j842f42696c\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c906de65-e3a2-4879-9236-7b176a8d1899\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c906de65-e3a2-4879-9236-7b176a8d1899\",\n \"name\": \"c906de65-e3a2-4879-9236-7b176a8d1899\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:38.123542143Z\",\n \"endTime\": \"2026-08-25T20:07:46.9158638Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8e5e0\u003e: \n failed to create HCP cluster np-downgrade-x827rx: failed waiting for cluster=\"np-downgrade-x827rx\" in resourcegroup=\"rg-np-downgrade-x827rx-59zprw89nzrh\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/601f12a6-e78d-4e43-816f-0a2f916a5e32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/601f12a6-e78d-4e43-816f-0a2f916a5e32\",\n \"name\": \"601f12a6-e78d-4e43-816f-0a2f916a5e32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:49.295862053Z\",\n \"endTime\": \"2026-08-25T20:07:56.866737842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8e5e0\u003e: \n failed to create HCP cluster np-downgrade-x827rx: failed waiting for cluster=\"np-downgrade-x827rx\" in resourcegroup=\"rg-np-downgrade-x827rx-59zprw89nzrh\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/601f12a6-e78d-4e43-816f-0a2f916a5e32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/601f12a6-e78d-4e43-816f-0a2f916a5e32\",\n \"name\": \"601f12a6-e78d-4e43-816f-0a2f916a5e32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:49.295862053Z\",\n \"endTime\": \"2026-08-25T20:07:56.866737842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"01e5501399c4ff9330ca8905c22cca453551f730bf7866919a23a1964d735140","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"071c52a55532188ca4d85bb7a66cc6fe5057e406feb7fa0eb18ba541618e910d","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"0b9d16275b3d7ff10a538398088cfd3b34d1f81c4cb68201208e8e4940acdb66","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"0bed7020ecd96a5cf05f223e463371381d77da2411a879375e1ceda63de28e4d","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"0ddc6a1225a6aa5e4af4f5f0cac4a1b2429896b3b2b8ecd24544d0bdee9d655e","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"137c0167738c3818a360a99bda994cbeeaf885b6a21dfa26c7f7e5381c79eb6a","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"150626fb402aa61e05e6c6f7e9bc8843969fdad7b45425051818fe06847d9754","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"15594220caf851c71a98818822ec8afd99496a6032084cc875914f18e55fca35","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"1b0503c4496d672dc2cfe86aa011acfb4fd95d2eec277298e9e02bf59cf65ab3","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"1babbd681ea180bcbac323e120929964548b008afb246353cbf1fad90e2045d0","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"227699a26b6d86c6ff9a362b2fd1395de244815866341bf1ceee8b2242c39b99","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"23e0583c28d9032ccce1bd36665f177c2f1db717f6e1cfd08f9ea2a16c2c6de9","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"2b7bfba4308decd6036acc7bc3a1df135b34ae217fab985b49db26b737a0d3a3","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"2bae653a8fb17447152d646aa0c2b4963ab31a22269313b92437db057bf036df","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"2c154aeee3d38aa0e65baa996b6067656fdcb76e0501965057b6de64078cad6d","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"37f401123d689a1b77a09fb3105cb5815a2f706e5e9877cf69d6ae68bf44f228","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"42009cf5afc6cd4364288595dd3fe884cb5f131f6903a745c42078e5d885c516","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"50e102a0b8a908cb91efb330c142288a7922ca39527756e228d95800edd32777","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"53f56986e4035859cdad5301f526d2a5f8fccefb68779c8f991838db02bdf8d5","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"5789fa442d4f8a0f4909b27b633de87680963e17d09512bd15731d2bd37905ee","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"5c886e13528881fff7670bd0116e318e6fdf76957dfb81580b348ec9602154a5","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"68545017c2b3bd81d0621bcf01c19e0de3848f03893c798a6c2aef8169f05af6","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"6bdc706667de6fd9046a69b69bd20920f987c7f6feff220cb363a388a2ebc362","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"6fe96d1ea335b0c185768786273c3d8084095250845c72ff17619116c26601e4","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"702d137fa1b1cc6cdf5e5a67bee5dcd22b5b57ed6578939ac00222e0d40e5187","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"86d19c78d0739357bd3f161b74003da30c80a9d54ee056afba56d5e92cfbbc8a","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"889088819d9dad08a48983198d0e17c9a48b8ac3d79ea908cbdb9bb35c082bc6","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"9bc356d5e4cf33cadfb9cc3dddabf59890f14d30c9b21e128d830f5bcd9b3092","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"a2da6b6b096cccb301a6ba7a740e8c3e59f518fd173797735af9ce7fdb6f0758","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"a6c110259b6db6815f32629ea3be7e1b59c5bf8c1cc0025a60f627f7d9a52eb7","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"ade895469bda3f6ff370f0428733134c672c1f332d14dbed05c869078e270076","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"b54b257434a08295e33fb0d9a84522989a2312ce08305e6584ba381ae359da3b","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"b67939caaf0419b44e1276fe638732fc70a0ba57b9c847d29eed62ccca5bd266","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"b933bdf4882baaf62138a22d1375d2afc64afb311d072ff04469393bbe10f8d1","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"bf05905ae686b69af2afa5478616503e24344e881559bb70f69d4c9570b0f7a8","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"c278280dc667ba84567309478370f71e50ba651e2a8a2399ea8b30d7333b71fe","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"c73655be7d93d003560cdc90a2bab9e176e6bcf65d8c644387408b36f348155e","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"d550d5cb6f50d0f9ff8f38ae93cfebf0a70e58547865e6dbf73e9cbe1c055656","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"d693677dcc2436c081b021447374afa98eb7158ee0d3dec68e19d233b84b5470","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"dee87de74156ea23b8b3520f533ef9190f72b7babcd20bcb8e2cf2a94d10dcdb","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"dfa42aba5c4cc2cef7d699b8c65ba25382242bb42c7e266ef350ef67bc80df08","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"e0b54c003aebec5d92b194a6f80ee906c128eb77169d7d2a7cba65e9aeeeba75","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"e5bdbec92086140fc310730009e23b06059442f01f5776e5f1192f7ae25b3f0f","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"e7f3a8d6fb04405286572a2d49c4ffc07ae39f040f9b3c52e8290ec4d6f9c045","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"e92d23b1277ca5f619831904feaa802051ec36b3b4c1e38928537ad0c359a4e7","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"f70d11b5e9cc7db70c1bd07ccd81a8b490b5e1aa926781339d876d128c6ce56b","pr_number":6681,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":36,"runs_affected":14,"failed_runs":14,"run_impact_percent":5.833333333333333,"anchor_occurrences":36,"after_last_push_count":13,"trend_counts":[0,0,36,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":36,"prior_last_seen_at":"2026-08-22T00:10:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-tlrb8m with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a7a5c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-tlrb8m: failed waiting for cluster=\"np-version-upgrade-cluster-tlrb8m\" in resourcegroup=\"rg-np-version-upgrade-tlrb8m-64c6wd9hqbps\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1769c72e-78e8-48ce-b214-b5a858f938c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1769c72e-78e8-48ce-b214-b5a858f938c5\",\n \"name\": \"1769c72e-78e8-48ce-b214-b5a858f938c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:26.310363981Z\",\n \"endTime\": \"2026-08-25T19:41:32.435006844Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-tlrb8m with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a7a5c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-tlrb8m: failed waiting for cluster=\"np-version-upgrade-cluster-tlrb8m\" in resourcegroup=\"rg-np-version-upgrade-tlrb8m-64c6wd9hqbps\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1769c72e-78e8-48ce-b214-b5a858f938c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1769c72e-78e8-48ce-b214-b5a858f938c5\",\n \"name\": \"1769c72e-78e8-48ce-b214-b5a858f938c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:26.310363981Z\",\n \"endTime\": \"2026-08-25T19:41:32.435006844Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-ttf5nhws49r9/cluster-candidate-4-20-fx47tw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a08e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-fx47tw: failed waiting for cluster=\"cluster-candidate-4-20-fx47tw\" in resourcegroup=\"rg-candidate-4-20-ttf5nhws49r9\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b05be7f-16c7-4109-9394-3dc217cc5c13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b05be7f-16c7-4109-9394-3dc217cc5c13\",\n \"name\": \"7b05be7f-16c7-4109-9394-3dc217cc5c13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:17.318805591Z\",\n \"endTime\": \"2026-08-25T19:42:23.583486998Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-ttf5nhws49r9/cluster-candidate-4-20-fx47tw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a08e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-fx47tw: failed waiting for cluster=\"cluster-candidate-4-20-fx47tw\" in resourcegroup=\"rg-candidate-4-20-ttf5nhws49r9\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b05be7f-16c7-4109-9394-3dc217cc5c13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b05be7f-16c7-4109-9394-3dc217cc5c13\",\n \"name\": \"7b05be7f-16c7-4109-9394-3dc217cc5c13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:17.318805591Z\",\n \"endTime\": \"2026-08-25T19:42:23.583486998Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f14340\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-p6gkdz5j5kdk\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cd1f359-f586-4859-b136-632a31d92bfd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cd1f359-f586-4859-b136-632a31d92bfd\",\n \"name\": \"3cd1f359-f586-4859-b136-632a31d92bfd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:27.431435186Z\",\n \"endTime\": \"2026-08-25T19:42:33.505289846Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f14340\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-p6gkdz5j5kdk\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cd1f359-f586-4859-b136-632a31d92bfd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cd1f359-f586-4859-b136-632a31d92bfd\",\n \"name\": \"3cd1f359-f586-4859-b136-632a31d92bfd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:27.431435186Z\",\n \"endTime\": \"2026-08-25T19:42:33.505289846Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"a2fca96d5c30f50d48abd3d91900db1407214ad4c57c888c781660c67cdbdfbe","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"1f94d94c8e70a8e7a8b56a5653939cf4935a72854bc9feda9124071da3fbfe56","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"2ce22b33b27841f4451ddbf18b57eaa0818348b0deaa93898d8394e7df7e56d0","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"30dad826455fa0938e7cec0e4f031d259f61105244ff995b4e802cf11d598f9a","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"c0dcf5c3ac2881b29fcaf6a19b5d8e99b79dd0936f8a80948a5bd717cba76449","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"521ed306a26b9f9dacf6c47c753c84342bde38e73eb4c5ef23908a8ca2f86d45","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"8a342e0125bf2802c9ea767d7ffb9709438a1f4e72f2ad64f7e052e12c236331","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"e51e9dfee4ba4c9daa2b79ed992ce895a67497a0787c0bbfe0188313ac862ed6","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"fdfa7061d24f7ad3a467bcd48be893b74bb83304ac5cba05b613cd11d788255d","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"236662ab2cf12e0855b7e39f0def5a11fc29e4cd526454501035df011b642ee5","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"c1d27f93ce1a22b612a32738d89512a0f0eca7cf6197d0e5f557147d4b9ce477","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","occurred_at":"2026-08-25T14:43:04Z","signature_id":"12cc269491c81e464e81232257f84d9ecbcb60190f84d4997ab10025e4d39380","pr_number":6676,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"489782a342d805c0338dec6f638d02703430b0e8f541b6a5142fbe16ac3c7ae8","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"93381f18ce633ba4d082796c83975ebeab65d65a6f8d890e5eb3fce8495ce24b","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"3c8445447816fab0494fb15002cb32f86c4fc7cc3167f611c626f8d9af1ff88e","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"3f0f7b3d38506c573a5570dd4fc273d4974d9a5328b2080e03b587e59e9463aa","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"0d1f7426cf173fcfa700b32a523143bec8203aa837293f2ac55c31004cab129a","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"99dd3763374e539920c960007cd1b14c1c67fd84b0f8ee47d35b2abd7ec3c4a2","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"82e0621d077685d0022747285dcb794c36f142bd42f8f8a172a77dcd55aad3e9","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"a12a937f6ae89e3736e75bd6e94756e9cbd733f875fec3c20ccc3e4632e8284a","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"d36de0542872df4f6d63fdbe3389505290d5b0baebed82cdd90fedf806ec3e01","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"431ff7e575eec123143334467cdb2866bc6f04d93674992976852f19ccc99809","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"4aece071fa661e61fc57264421a76c9eac13620f5e30bdc56e63af5b1984f5a1","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"983b22f1f6c46c18080a9b4eef0fb721f28445b2ab45c04695540c6e1542253d","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"d7b472469e4b347e99e5eb839d0b4c4d6750f5bc580fe678d89b0781d5908848","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"09789f559d5df6a2a46fb75a6ae925fba0fbd303143b135e8e8bf18f560ae90c","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"17edb3dd260ea059c244167666ed36031cba43f5f67e78698ae6697d0e919add","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"2b601911adad56c98c50dd26e4c497713a18ded9cd009a058cdef7ea37a3ab49","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"46cface1afceb818229ce74688d353ed8c60a22a002f15862ba902e03ea766aa","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"d0e594c3105542ca3fc8d2a4997071027f9acfcf8cc74b3eb6724ac1ff51a864","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"87423fb812ad03870c503443bdcecefa13a470d2f72adb08656f725ea5a893fd","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"9d9e7c31a0eab79ec1990c199b21ca4f9e3b8abb4900261837125436f88781d9","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"b6676f8c8a5cec645c6a41c81541d0d21fdc763921f2f8f5e84133e0d2a0b697","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"3be09b3efaa1d6639c2674bcbca81f107fd2e424dd0133709cfc55b04fc0a27c","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"48ff412681e68aef8f22697d9b84a39dbb80151cba4ee2ed83b622757bccfcd8","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092188089500831744","occurred_at":"2026-08-25T09:29:48Z","signature_id":"772bc28982c38b2a6bed58697050c102f7ccd329ca804037b2e6053a2681cbd4","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":34,"runs_affected":9,"failed_runs":9,"run_impact_percent":3.75,"anchor_occurrences":34,"after_last_push_count":4,"trend_counts":[0,0,34,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":19,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4j5sc54kln8q/cluster-candidate-4-22-sf6h7k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010e0360\u003e: \n failed to create HCP cluster cluster-candidate-4-22-sf6h7k: failed waiting for cluster=\"cluster-candidate-4-22-sf6h7k\" in resourcegroup=\"rg-candidate-4-22-4j5sc54kln8q\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de17841e-350a-4fc5-b9b8-7f89fd7f8e91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de17841e-350a-4fc5-b9b8-7f89fd7f8e91\",\n \"name\": \"de17841e-350a-4fc5-b9b8-7f89fd7f8e91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:37.268675469Z\",\n \"endTime\": \"2026-08-25T19:36:40.707957411Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, packageserver, community-operators-catalog, cluster-network-operator, cluster-storage-operator, certified-operators-catalog, cluster-policy-controller, openshift-route-controller-manager, dns-operator, catalog-operator, cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, openshift-controller-manager, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4j5sc54kln8q/cluster-candidate-4-22-sf6h7k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010e0360\u003e: \n failed to create HCP cluster cluster-candidate-4-22-sf6h7k: failed waiting for cluster=\"cluster-candidate-4-22-sf6h7k\" in resourcegroup=\"rg-candidate-4-22-4j5sc54kln8q\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de17841e-350a-4fc5-b9b8-7f89fd7f8e91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de17841e-350a-4fc5-b9b8-7f89fd7f8e91\",\n \"name\": \"de17841e-350a-4fc5-b9b8-7f89fd7f8e91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:37.268675469Z\",\n \"endTime\": \"2026-08-25T19:36:40.707957411Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, packageserver, community-operators-catalog, cluster-network-operator, cluster-storage-operator, certified-operators-catalog, cluster-policy-controller, openshift-route-controller-manager, dns-operator, catalog-operator, cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, openshift-controller-manager, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-grhvwwgnmxfw/cluster-candidate-4-20-s6k958 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111c140\u003e: \n failed to create HCP cluster cluster-candidate-4-20-s6k958: failed waiting for cluster=\"cluster-candidate-4-20-s6k958\" in resourcegroup=\"rg-candidate-4-20-grhvwwgnmxfw\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2555a7c-e636-4588-b48a-b1225e2d8f06\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2555a7c-e636-4588-b48a-b1225e2d8f06\",\n \"name\": \"d2555a7c-e636-4588-b48a-b1225e2d8f06\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:38.527859766Z\",\n \"endTime\": \"2026-08-25T16:40:38.616570204Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ignition-server, ignition-server-proxy, certified-operators-catalog, catalog-operator, control-plane-pki-operator, kube-controller-manager, kube-scheduler, cluster-network-operator, cluster-node-tuning-operator, cluster-version-operator, dns-operator, cluster-image-registry-operator, konnectivity-agent, redhat-operators-catalog, machine-approver, openshift-apiserver, openshift-controller-manager, ingress-operator, community-operators-catalog, cluster-policy-controller, openshift-route-controller-manager, cluster-storage-operator, olm-operator, azure-cloud-controller-manager, packageserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, kube-controller-manager deployment has 2 unavailable replicas, kube-scheduler deployment has 2 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-grhvwwgnmxfw/cluster-candidate-4-20-s6k958 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111c140\u003e: \n failed to create HCP cluster cluster-candidate-4-20-s6k958: failed waiting for cluster=\"cluster-candidate-4-20-s6k958\" in resourcegroup=\"rg-candidate-4-20-grhvwwgnmxfw\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2555a7c-e636-4588-b48a-b1225e2d8f06\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2555a7c-e636-4588-b48a-b1225e2d8f06\",\n \"name\": \"d2555a7c-e636-4588-b48a-b1225e2d8f06\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:38.527859766Z\",\n \"endTime\": \"2026-08-25T16:40:38.616570204Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ignition-server, ignition-server-proxy, certified-operators-catalog, catalog-operator, control-plane-pki-operator, kube-controller-manager, kube-scheduler, cluster-network-operator, cluster-node-tuning-operator, cluster-version-operator, dns-operator, cluster-image-registry-operator, konnectivity-agent, redhat-operators-catalog, machine-approver, openshift-apiserver, openshift-controller-manager, ingress-operator, community-operators-catalog, cluster-policy-controller, openshift-route-controller-manager, cluster-storage-operator, olm-operator, azure-cloud-controller-manager, packageserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, kube-controller-manager deployment has 2 unavailable replicas, kube-scheduler deployment has 2 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee2ca0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-bb6s68gx7kh6\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9561d49c-5877-4d79-855f-4df08a7e3fae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9561d49c-5877-4d79-855f-4df08a7e3fae\",\n \"name\": \"9561d49c-5877-4d79-855f-4df08a7e3fae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:40.989997281Z\",\n \"endTime\": \"2026-08-25T16:40:48.583774176Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, olm-operator, cluster-version-operator, hosted-cluster-config-operator, cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, catalog-operator, cluster-policy-controller, packageserver, community-operators-catalog, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee2ca0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-bb6s68gx7kh6\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9561d49c-5877-4d79-855f-4df08a7e3fae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9561d49c-5877-4d79-855f-4df08a7e3fae\",\n \"name\": \"9561d49c-5877-4d79-855f-4df08a7e3fae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:40.989997281Z\",\n \"endTime\": \"2026-08-25T16:40:48.583774176Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, olm-operator, cluster-version-operator, hosted-cluster-config-operator, cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, catalog-operator, cluster-policy-controller, packageserver, community-operators-catalog, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"f5fbd5480e218821391ad606bc369871f81ab79ca13513e6166733cd7e61e250","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"1f2d818f791aabf7ca966417dcc515df6d86223731a47e82cb83edd0e59ba4f7","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"2e500e0b56bd09e2e3065c06d8cad98af6fc49618dd7b23cbffb2625133fb624","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"6d79c60b75c7247e2641f3408daf5532b9c4fe3a7b5290b37b8be3c077c8fcb6","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"62e1bcec648cf6ca85b4778d6a217f22d6ca9de165a453fb96f3ba95214f5341","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"69a7645ef69f7c63a48191f6533d04ca0a326a56df092b3558b2f664fafc5a80","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"86c42132d5d5019c3e30fa7d9fcde1ed01362afa33a0740202359c037a9d6aba","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"45742a02b7e0c32d86b97863f4ea299dc6196292e7ab7de047ae62dec5023312","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"722d515e29fb9ec826abf197c41290ecc658816cb59a8a63c0cebddb0232cee7","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"90684796a4aec3250755dd0c0757e80b9dd9a4d843996fedaf91a968864f7cc1","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"af3ee147c3245ccf45bdc2deacd994348ca6f9c719f39a02ed738a5c4ec6e6d7","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"e814323bff83120ddf30b4df3d591da6897add0520ab77b9aece5773322a790f","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"25b082e6f3fccd0eeb0248caee3439349733372ff0b9b1c11e1a4ee3a27cb02e","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"25be28244289ac46a2d410d588e4df19d0867d5c662db6e3ba488825169d0ea1","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"d17d79e8ac084d555f983d2abb36810032fbe5d1d3056157e89dc068b55ec851","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"26682ebc65e6771f4deee3248fa461bb0de542a4ab0eebfc70762ad29bf4f7f8","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"32b3f6d0980058b52ca8a27936b8fa3d683b85efd75564d377af68c930a114db","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"504ae63c8f089cb82a68d41adbc6f1b843f3855c8b1b37f224907c01e60811f8","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"812cfe1df0bd2944c0ae39c8f4dc119f0c86e898e12e7120fca50c7be2fd1a6a","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"8366f1f3e151dc6dc562cd5621b250932345234159c653b20f33f1095b4d2763","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"1e1bd91e2da41c37dab2549990d0b6c3580daa21ab6ad95e1315e6a8a8df6173","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"429e22ec2d21dbf61e0c7b6393566a722dc292f14b64db5f2803331be42369e0","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"732bac21dc6f40ccf236a6b2bee56772eb8544e2c5358966c3a2a55ba78a23ba","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"84df8ea31194f1a34ce54d507db6d896e44775caf63f19fedd4bd03f18b43d1b","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"a8c7c70bd3d6dd8630d347b4d24584ab9920ea0b8c33dde6ba59740ac0218935","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"f070923f4a7c1696bb4766ac883978b74411c5db4f343ca2e398a5b72fdf0f61","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"283564a0bebb53ed7bfe47fcf31d4874c7bd4b8c4a658e0aea01d9dd41b0242d","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"7a249edbd9da943618f89813673eaaae45051d2a658fe6c77b5dd31a98bcbc7a","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"8a57d6dcc713db3bc6dcc797a3a204959181e3f6745d603dd5d1db0fbf1450de","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"9e89748c5a442a7a3524b941a931bf754405a837847407c813dd70e1c49eea38","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"b2468c4e9d126aa56394e8ef9b7bce10bb3f3907f961e58663665d48e197b818","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"dcaf8967a5afa5f20ee5c32afe78d8acf920baee1936ceb3553c7140fba63e19","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"e27b3eb99bc3107ba8d48dd22285100fd839e258993d8e1be4af2f04735bbfd0","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"e3bec1f2cb1a89b8221c4f2e37ae1979157dfc028f4587eae931e2b14a591671","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":33,"runs_affected":20,"failed_runs":20,"run_impact_percent":8.333333333333334,"anchor_occurrences":33,"after_last_push_count":10,"trend_counts":[0,4,29,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":28,"prior_last_seen_at":"2026-08-22T00:10:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0007ab4d0\u003e: \n GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/805512f7-5822-4d4a-b7f2-904c7a51afce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/805512f7-5822-4d4a-b7f2-904c7a51afce\",\n \"name\": \"805512f7-5822-4d4a-b7f2-904c7a51afce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:32.391153149Z\",\n \"endTime\": \"2026-08-25T19:39:32.410048712Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0007ab4d0\u003e: \n GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/805512f7-5822-4d4a-b7f2-904c7a51afce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/805512f7-5822-4d4a-b7f2-904c7a51afce\",\n \"name\": \"805512f7-5822-4d4a-b7f2-904c7a51afce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:32.391153149Z\",\n \"endTime\": \"2026-08-25T19:39:32.410048712Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fae120\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pqxcnq8z9jpd\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd31e0f1-3a30-4284-a7ce-0a370221b212\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd31e0f1-3a30-4284-a7ce-0a370221b212\",\n \"name\": \"bd31e0f1-3a30-4284-a7ce-0a370221b212\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:31.984108593Z\",\n \"endTime\": \"2026-08-25T19:39:32.326305358Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fae120\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pqxcnq8z9jpd\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd31e0f1-3a30-4284-a7ce-0a370221b212\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd31e0f1-3a30-4284-a7ce-0a370221b212\",\n \"name\": \"bd31e0f1-3a30-4284-a7ce-0a370221b212\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:31.984108593Z\",\n \"endTime\": \"2026-08-25T19:39:32.326305358Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019ed80\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lxqrtnj4sd4l\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bb31d6b-0f83-4e8f-801b-d83aa356a651\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bb31d6b-0f83-4e8f-801b-d83aa356a651\",\n \"name\": \"6bb31d6b-0f83-4e8f-801b-d83aa356a651\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:28.693998542Z\",\n \"endTime\": \"2026-08-25T19:39:32.413356519Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019ed80\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lxqrtnj4sd4l\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bb31d6b-0f83-4e8f-801b-d83aa356a651\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bb31d6b-0f83-4e8f-801b-d83aa356a651\",\n \"name\": \"6bb31d6b-0f83-4e8f-801b-d83aa356a651\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:28.693998542Z\",\n \"endTime\": \"2026-08-25T19:39:32.413356519Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"1a10e9f0d0ebd327c61e8e899905b12b28ad26f43e7f89a4aa098b09ef35fc21","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"569936c5ee5c9e4cdd5de51f80b6ee2ec197ed73598a8d190cbd9e81f0f34f1b","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"bf78b2adb206b76ebe7b562c8f5d0dcff4cb6f69f3835690af34a77d874060a1","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"c9efb511f1cb5ecb77052682b34bef74a06a5d809c163ae9d53ba77fc8559f19","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"7e61cb838af28069775e4047abe596d948ba92c34b9bf7479496b56271f567b9","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"a5efaacf08081a9c1c8f925d50e651cf29e6bdbb45a3190694979b6ed844c618","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"8467082f94f1cddceb7436a951e2eb6e01b80042e0072d9c5e4e2e131f794e55","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"daeffcbe55429e419c5333569b8df49743acfca0ee5ac2e58304d2421599fcfb","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"383b14a099f2ce5791796cf71b4050d10ad8f6db01c076f62c792d982c8b659a","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","signature_id":"320a0a462e18ca42da7b1b26ae6a21190926e34c2244492c5c4e0a1c81102a82","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"148bb476ee1b7952c4359b6ada2341d8424d4da89438cb857f8b6bff27286eb4","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"0238da5b4ee608676a62f88216ac5a7d91c16da91f1ec76842258c4d72bebf4e","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"19f2511c94e5b6ca21cc71bd63aabc9053c7c653b42b617ec879ca395e30580c","pr_number":6630,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"6cd3b4ec3a78c5076ca55dacbf68ec344e1698246fdf7d2fed60e6b4e6630276","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"31be970aed5c1350ca97847fcd67fc79e890462b95ed0fd2c70eb16c601cc4af","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"da7c4bc1633eca36853e5843a03bffeb0f874d7d6c7c76694adad89844da3845","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"3397e3c177e331b0b3c56e3aca57d39b4697fcc3c0632de87b50427da89cf9dd","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"301d0f6d3f61c8fc4bfdf4b36d3da0f6a5670b9ba6b385d3c18bb6644f1afcaf","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"e70a8923551947fd69aed4662e45c5ae1a351669ab5e6230b53b2fb9b06f69bf","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"29bc2f93a6094b4a5d5cf641b428320d4890242a1af692aabdd89032ec92bca7","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"66a0672a0448bcbf1d60a1d5cafd7b371b74508101d9fc34d0777b13e4cf76e2","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"1c7bb16e4b644044bda0b75b866f85ba0a96450750139ae786fdc9571148e984","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"fd5838277e9b2418a0afbc2057c3ce5fc4bd346774696e4589e2c05f45f73f1e","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"09f9bb566a2cac0401d2761a3e203049bfc5d3f036f5254838e3a2c316495d34","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"0fc0765dde4e73e193cda291461bd615b21c604ca712b5db69f701c1f66f193e","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"313c6b86047e9e386b672d793f4b878d127f3c65a84326d4cc2a6231581f8bc6","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"643ac15ebd4f500d7cdfd1c31ab51409301a12ee67319d5fe193f4ec82a3edb7","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"b2a39dd5fceed70e654ab8c04be8e55e67bd4e3e353048a2a56cb92b2721b5ad","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"f5dd8ef4e673ae5c7d050ce26d2eedefb62e36c6ff1f4f21e2c53c0420687784","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"e79e05105a6dd37d39aa319e6b82dc1c85c13b7322507d230499b9d98521e94c","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"7f0730f0d1c7e0eb7a035ce2e1a824a7fb99f16bf281917ec8e0ea592b939bdd","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"f45d8f4fc578e9bcbc89c15b0ea557b08d4323587837d33c561c269a0b539be9","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"f749c71c85c572aa2c57394be8a931e71c30eda717753e81fa6456d997c9c5fc","pr_number":6658,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":28,"runs_affected":28,"failed_runs":28,"run_impact_percent":11.666666666666666,"anchor_occurrences":28,"after_last_push_count":7,"trend_counts":[0,28,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":34,"prior_last_seen_at":"2026-08-21T03:51:03Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":6},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":4},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":18}],"full_error_samples":["time=2026-08-24T13:26:56.540Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:26:58.762Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T13:27:00.485Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=abeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8084480-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fabeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969\ntime=2026-08-24T13:35:32.482Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/abeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969/operationStatuses/08584140288659502671\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/cluster-kyv757fphd2fm\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j8084480-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-24652102-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-24652102-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-24T13:31:41.774Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:31:42.863Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T13:31:44.020Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=cf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5557248-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5\ntime=2026-08-24T13:39:15.842Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/cf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5/operationStatuses/08584140285820210806\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/cluster-ce5kkcjcnxbu2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5557248-mgmt-2/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-20013346-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-20013346-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-24T13:26:28.921Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:26:30.957Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T13:26:32.602Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2132480-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848\ntime=2026-08-24T13:34:04.585Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848/operationStatuses/08584140288938416315\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/cluster-d54rkmqgu5xlq\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2132480-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-42034125-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-42034125-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091873075288084480","occurred_at":"2026-08-24T12:59:40Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872721045557248","occurred_at":"2026-08-24T12:58:16Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872693082132480","occurred_at":"2026-08-24T12:58:09Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872666397970432","occurred_at":"2026-08-24T12:58:03Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091868107806085120","occurred_at":"2026-08-24T12:39:55Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6585,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091867251824136192","occurred_at":"2026-08-24T12:36:32Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6585,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091856057667162112","occurred_at":"2026-08-24T11:52:03Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6585,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091852685220253696","occurred_at":"2026-08-24T11:34:18Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6424,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844440330080256","occurred_at":"2026-08-24T11:05:53Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844377495212032","occurred_at":"2026-08-24T11:05:38Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844284889174016","occurred_at":"2026-08-24T11:05:16Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091840235057975296","occurred_at":"2026-08-24T10:49:11Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091827867787726848","occurred_at":"2026-08-24T10:00:02Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091827277728845824","occurred_at":"2026-08-24T09:57:41Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6429,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091823405572558848","occurred_at":"2026-08-24T09:42:18Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091816375096971264","occurred_at":"2026-08-24T09:14:22Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6429,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091814034880860160","occurred_at":"2026-08-24T09:05:04Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6517,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091812979992432640","occurred_at":"2026-08-24T09:00:53Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091803125458407424","occurred_at":"2026-08-24T08:21:34Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091801419551084544","occurred_at":"2026-08-24T08:14:51Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091795137024757760","occurred_at":"2026-08-24T07:49:58Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091792609700745216","occurred_at":"2026-08-24T07:39:56Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6278,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6656/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091776267920609280","occurred_at":"2026-08-24T06:35:00Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6656,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091774664165560320","occurred_at":"2026-08-24T06:28:37Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091750724990930944","occurred_at":"2026-08-24T04:53:30Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091747201922371584","occurred_at":"2026-08-24T04:39:30Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091734160543584256","occurred_at":"2026-08-24T03:47:41Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091733541242015744","occurred_at":"2026-08-24T03:45:13Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6646,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":25,"runs_affected":13,"failed_runs":13,"run_impact_percent":5.416666666666667,"anchor_occurrences":25,"after_last_push_count":15,"trend_counts":[0,2,23,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":57,"prior_last_seen_at":"2026-08-22T00:13:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e46360\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjpc76rq228c\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812b9f46-3042-442f-9212-72e785e2f21e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812b9f46-3042-442f-9212-72e785e2f21e\",\n \"name\": \"812b9f46-3042-442f-9212-72e785e2f21e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:26:56.464409042Z\",\n \"endTime\": \"2026-08-25T19:46:01.698082602Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e46360\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjpc76rq228c\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812b9f46-3042-442f-9212-72e785e2f21e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812b9f46-3042-442f-9212-72e785e2f21e\",\n \"name\": \"812b9f46-3042-442f-9212-72e785e2f21e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:26:56.464409042Z\",\n \"endTime\": \"2026-08-25T19:46:01.698082602Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019edc0\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-prhmmrf56mn9\" to finish creating: GET https://rp.j0715520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90e4482a-a8a6-47af-8a58-d4c088a6709f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90e4482a-a8a6-47af-8a58-d4c088a6709f\",\n \"name\": \"90e4482a-a8a6-47af-8a58-d4c088a6709f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:35:42.014090967Z\",\n \"endTime\": \"2026-08-25T14:54:47.709528737Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019edc0\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-prhmmrf56mn9\" to finish creating: GET https://rp.j0715520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90e4482a-a8a6-47af-8a58-d4c088a6709f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90e4482a-a8a6-47af-8a58-d4c088a6709f\",\n \"name\": \"90e4482a-a8a6-47af-8a58-d4c088a6709f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:35:42.014090967Z\",\n \"endTime\": \"2026-08-25T14:54:47.709528737Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:739]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004c9840\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-wpzzs6\" in resourcegroup=\"rg-np-dg-minor-wpzzs6-6jql8zwjxxlb\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73b2139d-e941-4022-8bf3-72abae50ef1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73b2139d-e941-4022-8bf3-72abae50ef1a\",\n \"name\": \"73b2139d-e941-4022-8bf3-72abae50ef1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:35:50.588975119Z\",\n \"endTime\": \"2026-08-25T14:54:53.875483949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:739]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004c9840\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-wpzzs6\" in resourcegroup=\"rg-np-dg-minor-wpzzs6-6jql8zwjxxlb\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73b2139d-e941-4022-8bf3-72abae50ef1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73b2139d-e941-4022-8bf3-72abae50ef1a\",\n \"name\": \"73b2139d-e941-4022-8bf3-72abae50ef1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:35:50.588975119Z\",\n \"endTime\": \"2026-08-25T14:54:53.875483949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"0f5a2599c879fd0d600773df180f9eb5e17770f7cbb1109f8e5936709c16bd9c","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","occurred_at":"2026-08-25T13:34:16Z","signature_id":"52544ef6923df629cf38fa8f3d327d749d9b4fc26a68706642aec73ba7646ec7","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"44ad071145b6440bebd0a2dc3f06efae0013a7f9125c8d5d38fc3bea9f263863","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"2ee84b2de684d6bebbf318c8d5de7d1297cf8cffb2cae88ad176fe02897a3e46","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"3ee97bfc9952749cbcf036e0d8407c3873b852dade18372153b3dfc59818dac8","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"8d522cd2839df39682d21c0af249f6118f11366a2bf105536db9eb68cb5a9451","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092234666756018176","occurred_at":"2026-08-25T12:56:26Z","signature_id":"fba360027133779a17d7b5f7a04eb5d7f56e70b4761fe51394db3cfa7dded15e","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"0f33bde12ddadf9e7c9bfc135e1acbf84861720d826fda54f0ad89d0271fe465","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"1e5d4e64aee3bb0b78e1fb3308473036771fbfa7bf224bebfc3e1ddb47aca9c8","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"5ac2db27048619c0a83b9d739f995196c79b0762abf056bc200ed007d3557d7c","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"7ba4d18cd6273c16cfe80bffe517d25785f0d11a2228c4383f25ed32d2c438d6","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"96854033898edccbf2bb6b496287f357219f96280efefccded408cffb3a2f090","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"4806e756ed0578671bc1be64618a268b2aa1f613e22beef835379c8a977148f3","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"da59667e1510570b8a7920b7fcdf7406c1b2daaf77247d80291d5db3549304d3","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"eb35914289e36ada82afb95d0842ac1cbf05cda80bee90852c8426b9626b9565","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"0ac0ed3b0a9732a181325ecdaa144e0e29914071fc6f1496b3b0c6f22509cdba","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"806a6cee782cfb1e97e02367c95a679ce5b8a00edfd0c58d876dd15273163cad","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"74b264555e08507bc1b8cd31ab8e6916b262467b913f5056245709dac63bf473","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"84726c82ed97016f12844a47373fafc8ccb52c61c42bd82f0b0f6c4092b047bd","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"9862cf1db12a585c6af07719f78d5cb393fa5c03f36fe7693b11f8ee1e06cab2","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"e3ff43e82a0f930ac10e5802807ad5e255f69a88161d6322c6013bce16cd034d","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"f8dc4e77835e39b0b6cc3a84dbc11fd7f2cbf76af6acd98523b7cdc62c8c4941","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"a9fc0ce0c34af3d53b841c0d03266056bd51cbfcda7aed993a0a0b308252dad0","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977503907254272","occurred_at":"2026-08-24T19:54:38Z","signature_id":"eb508bccb119e029161a871dee87adcf17a97ac2d0acf65b89eac6aede0f896a","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"29b0688020a02097d8409be35d56bc8e657d156f249e7e0111a076d22256e797","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":23,"runs_affected":20,"failed_runs":20,"run_impact_percent":8.333333333333334,"anchor_occurrences":23,"after_last_push_count":109,"also_in":["STG"],"trend_counts":[0,4,19,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":164,"prior_last_seen_at":"2026-08-22T00:26:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abe4c0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-b7hpj2n7vp57\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6613b466-27ca-4523-a803-29e6469bc399\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6613b466-27ca-4523-a803-29e6469bc399\",\n \"name\": \"6613b466-27ca-4523-a803-29e6469bc399\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:41.448636527Z\",\n \"endTime\": \"2026-08-25T19:39:42.477039993Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abe4c0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-b7hpj2n7vp57\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6613b466-27ca-4523-a803-29e6469bc399\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6613b466-27ca-4523-a803-29e6469bc399\",\n \"name\": \"6613b466-27ca-4523-a803-29e6469bc399\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:41.448636527Z\",\n \"endTime\": \"2026-08-25T19:39:42.477039993Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-wwz9mh at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8e6c0\u003e: \n failed to create HCP cluster np-skip-minor-wwz9mh: failed waiting for cluster=\"np-skip-minor-wwz9mh\" in resourcegroup=\"rg-np-skip-minor-wwz9mh-j759r47mqgx8\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af4a457c-c413-4047-bdb1-2c68cdac3a98\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af4a457c-c413-4047-bdb1-2c68cdac3a98\",\n \"name\": \"af4a457c-c413-4047-bdb1-2c68cdac3a98\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:50.487713455Z\",\n \"endTime\": \"2026-08-25T19:36:50.727476187Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-wwz9mh at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8e6c0\u003e: \n failed to create HCP cluster np-skip-minor-wwz9mh: failed waiting for cluster=\"np-skip-minor-wwz9mh\" in resourcegroup=\"rg-np-skip-minor-wwz9mh-j759r47mqgx8\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af4a457c-c413-4047-bdb1-2c68cdac3a98\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af4a457c-c413-4047-bdb1-2c68cdac3a98\",\n \"name\": \"af4a457c-c413-4047-bdb1-2c68cdac3a98\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:50.487713455Z\",\n \"endTime\": \"2026-08-25T19:36:50.727476187Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6nrgxcz8znbm/cluster-candidate-5-0-bz6rh7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001044860\u003e: \n failed to create HCP cluster cluster-candidate-5-0-bz6rh7: failed waiting for cluster=\"cluster-candidate-5-0-bz6rh7\" in resourcegroup=\"rg-candidate-5-0-6nrgxcz8znbm\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0e826bb-86ad-484d-940a-a69d7c037731\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0e826bb-86ad-484d-940a-a69d7c037731\",\n \"name\": \"c0e826bb-86ad-484d-940a-a69d7c037731\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:17.696794179Z\",\n \"endTime\": \"2026-08-25T16:40:18.610816463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6nrgxcz8znbm/cluster-candidate-5-0-bz6rh7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001044860\u003e: \n failed to create HCP cluster cluster-candidate-5-0-bz6rh7: failed waiting for cluster=\"cluster-candidate-5-0-bz6rh7\" in resourcegroup=\"rg-candidate-5-0-6nrgxcz8znbm\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0e826bb-86ad-484d-940a-a69d7c037731\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0e826bb-86ad-484d-940a-a69d7c037731\",\n \"name\": \"c0e826bb-86ad-484d-940a-a69d7c037731\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:17.696794179Z\",\n \"endTime\": \"2026-08-25T16:40:18.610816463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"b0b6ea785c6adb804d2ab49b2e9c929c0658e46f9052aac84f220ac1f23f4ce1","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"e6e6771b32f676dac7dd06703c5d639fb506b0db753539a9e145282b66c90963","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"7de4e86735efd5a13296026c22a5cfa492755634aa30ae6440c336d9757f0cba","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"7afa6a3cb0a5460454f74e2a2b1999dd33e3f60965567464435dde991505b283","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092257335450800128","occurred_at":"2026-08-25T14:26:35Z","signature_id":"f0c15c8af4ed67bd702e14b46d290c6f294cf901457e9886c4b4f48671c2cecc","pr_number":6669,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"672eae11be59af76cbd1f197619054a7417629804d462d7487cf8b975b64fdd2","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"a62be0324e4af4aa551666eb60b0b9159ceb611d57868bbf7b768ffe258fbfbc","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"d5aeb36659e2fe6d42bc3c374436c19a2f23ca86b7d193e2d4e494e04f3f5eda","pr_number":6630,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","occurred_at":"2026-08-25T13:34:16Z","signature_id":"5310caabe4a3b378f21ce354c7c0c3aac76e1ac8d392c8d5eaecc0741c517da0","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"09a91b6d59e4fbee757881e9171ec763df5844bb16934da2b1be95a803e452f3","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092215603107467264","occurred_at":"2026-08-25T11:40:45Z","signature_id":"2aaa34c36208117bcee63f75dc471d9e9468caff13d1659c2583035ece83c2fb","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"b43fff9cd9bd036194d64006e2bedb42a5de47a08b4bb6a5327c7a329e3576da","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"ed7279f8680b41f53bf9dea3800ff6d209dab4158b53dc3aa29303a69736db57","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"befa32cb5bdc16812833dab6b4ef32700219a3216e83ebcad65e2edc52d10b6c","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"8cb8819d08e22597597cadf9ee8ec483916ec2e91ed7308d6b08f79ac3c67f38","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"a7bdda2906b7a00d3ea96fe2c15e55467a147dabf9881ad88f1b9c2804a5d6af","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"e006fc8de127b51c0b9862eaa26e5d9a844309286eb194166ccd19b3c55c024d","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146681486249984","occurred_at":"2026-08-25T07:06:53Z","signature_id":"0d69566e1be51af62861a571d03269f04a5db50727a3f7e9802a7d8a5eb2ef75","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092142908357480448","occurred_at":"2026-08-25T06:51:54Z","signature_id":"4afdddbae344462c995de2251551a735a47164d0bd4bc3714ba035dce8cb03a0","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"409a88ec07ceec5d40d5ae23f780d5c74cad626824811548eedaa155d1eba475","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"3605097c99e7a1623e2e3dd91c5b9641833e016d1c0c516d9b0275085f65863f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"4de409ed2fffcc46ddb557100c947bf3b037028f8c648ac70391763c58835df0","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"2cc87e67c265ba4f74a9a313e8f38a6f54437455cae5243f5e1f2cc3556cdfc9","pr_number":6658,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":17,"runs_affected":17,"failed_runs":17,"run_impact_percent":7.083333333333333,"anchor_occurrences":17,"after_last_push_count":11,"trend_counts":[0,6,11,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":65,"prior_last_seen_at":"2026-08-22T04:44:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":17}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"371e2dd52ef73c97161118e4d4efe23b9ace8422b8af743704455c3d7faec23b","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6630,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6677,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240300545150976","occurred_at":"2026-08-25T13:18:54Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197879035727872","occurred_at":"2026-08-25T10:30:20Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985911028912128","occurred_at":"2026-08-24T20:28:03Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"371e2dd52ef73c97161118e4d4efe23b9ace8422b8af743704455c3d7faec23b","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":15,"runs_affected":9,"failed_runs":9,"run_impact_percent":3.75,"anchor_occurrences":15,"after_last_push_count":0,"trend_counts":[0,10,5,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008ea800\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wpt5bd5wpfv7\" to finish creating: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/553b9e0e-f314-4ed2-a644-3fd009bac34c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/553b9e0e-f314-4ed2-a644-3fd009bac34c\",\n \"name\": \"553b9e0e-f314-4ed2-a644-3fd009bac34c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:22:31.343342864Z\",\n \"endTime\": \"2026-08-25T20:41:33.231859597Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008ea800\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wpt5bd5wpfv7\" to finish creating: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/553b9e0e-f314-4ed2-a644-3fd009bac34c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/553b9e0e-f314-4ed2-a644-3fd009bac34c\",\n \"name\": \"553b9e0e-f314-4ed2-a644-3fd009bac34c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:22:31.343342864Z\",\n \"endTime\": \"2026-08-25T20:41:33.231859597Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001022220\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-dm2lv9wtz9pd\" to finish creating: GET https://rp.j9302656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0f240228-e711-46ec-8078-6d29993f3656\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0f240228-e711-46ec-8078-6d29993f3656\",\n \"name\": \"0f240228-e711-46ec-8078-6d29993f3656\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:54:14.698555112Z\",\n \"endTime\": \"2026-08-25T19:13:18.75896335Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001022220\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-dm2lv9wtz9pd\" to finish creating: GET https://rp.j9302656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0f240228-e711-46ec-8078-6d29993f3656\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0f240228-e711-46ec-8078-6d29993f3656\",\n \"name\": \"0f240228-e711-46ec-8078-6d29993f3656\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:54:14.698555112Z\",\n \"endTime\": \"2026-08-25T19:13:18.75896335Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00041c4e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6bdpbrgzdzc8\" to finish creating: GET https://rp.j9832320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccc072ba-d2dd-4a86-b29f-6086f60cb482\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccc072ba-d2dd-4a86-b29f-6086f60cb482\",\n \"name\": \"ccc072ba-d2dd-4a86-b29f-6086f60cb482\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T17:41:06.665554524Z\",\n \"endTime\": \"2026-08-25T18:00:10.653150122Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00041c4e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6bdpbrgzdzc8\" to finish creating: GET https://rp.j9832320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccc072ba-d2dd-4a86-b29f-6086f60cb482\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccc072ba-d2dd-4a86-b29f-6086f60cb482\",\n \"name\": \"ccc072ba-d2dd-4a86-b29f-6086f60cb482\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T17:41:06.665554524Z\",\n \"endTime\": \"2026-08-25T18:00:10.653150122Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092336210943938560","occurred_at":"2026-08-25T19:39:54Z","signature_id":"575bcdf143a3e1cfb87437c6e889db059f51d09aaf88f43d9f5ba29604ae11bb","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311337039302656","occurred_at":"2026-08-25T18:01:04Z","signature_id":"970a4e186dca3e58eb5fc30dda68bc89be799ddbb9763794ffcc3be928d20bb8","pr_number":6639,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","occurred_at":"2026-08-25T16:56:19Z","signature_id":"0b8690595b1dfed6e36bd3e52f32e3625ed1e752008ebb3edcf9215308afc984","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"473e68c054364e4d1143001e092d3c5f19b78d56a2f0385244813ef7b9b36807","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"ab35ad7b76b266586beb627cd857234f4deca4448950461a1483290474e1d37e","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"87055b770927de3226a00a72e6a1e8f425c4181b7305ba9fd94502a36df5fec7","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"e5919a5aa6e234b2c6b16539e4375d2ff9f7724d31d00cb9d8f2c6f3bd3b38ad","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"4bba4cf482e3fa39dfa2ce3e799d7eb2c477fcc7e6ad0b0350124af0d131c053","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"5a1d95e1ce2afb3cd1318374aea17f4a6e4f5d30ef03f9f5a0cc441e9043a771","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"28a8b869f35a9cd1794a8e197fc2255f15a11da6d73b85987e232efa1871d3dc","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"2cae48dc9a7ac77e5a8ac54a32e19e04d69dc07b4bbfdffac675fbb83525b59a","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"51d9d625d26042275bccf29b990ed16b6ab0c82bab381da48ae1943a65da776a","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"57e93beced19ad75ed8576f3f0efd935e4fd8431d2ed07db01840f4e2fefa928","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"9137a39c80626f35b65cbd5b530df9d8f5ceed8a3ed523d23d20fe02b5217e4f","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"0119ef26d501844a6f69a87b74e13b68dec77db9c3af742f0d6e0ca5c74e1853","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":13,"runs_affected":9,"failed_runs":9,"run_impact_percent":3.75,"anchor_occurrences":13,"after_last_push_count":1,"trend_counts":[0,0,13,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":15,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d808a0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-m46xgd2l9dwf\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93218d8d-7467-45c4-ba19-ca9ca2bada13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93218d8d-7467-45c4-ba19-ca9ca2bada13\",\n \"name\": \"93218d8d-7467-45c4-ba19-ca9ca2bada13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:21.65358047Z\",\n \"endTime\": \"2026-08-25T19:39:22.258875158Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d808a0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-m46xgd2l9dwf\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93218d8d-7467-45c4-ba19-ca9ca2bada13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93218d8d-7467-45c4-ba19-ca9ca2bada13\",\n \"name\": \"93218d8d-7467-45c4-ba19-ca9ca2bada13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:21.65358047Z\",\n \"endTime\": \"2026-08-25T19:39:22.258875158Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004e2c60\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fzg7bs5ws6ct\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/444c3209-4e2e-4761-820a-0c2926e22f5c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/444c3209-4e2e-4761-820a-0c2926e22f5c\",\n \"name\": \"444c3209-4e2e-4761-820a-0c2926e22f5c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:29.031939015Z\",\n \"endTime\": \"2026-08-25T19:36:30.68458199Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004e2c60\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fzg7bs5ws6ct\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/444c3209-4e2e-4761-820a-0c2926e22f5c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/444c3209-4e2e-4761-820a-0c2926e22f5c\",\n \"name\": \"444c3209-4e2e-4761-820a-0c2926e22f5c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:29.031939015Z\",\n \"endTime\": \"2026-08-25T19:36:30.68458199Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fc5c0\u003e: \n failed to create HCP cluster np-downgrade-9wkwgd: failed waiting for cluster=\"np-downgrade-9wkwgd\" in resourcegroup=\"rg-np-downgrade-9wkwgd-c6w4l726nv9w\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4ede516-845b-4e65-988d-164f84fc3105\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4ede516-845b-4e65-988d-164f84fc3105\",\n \"name\": \"b4ede516-845b-4e65-988d-164f84fc3105\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.55170955Z\",\n \"endTime\": \"2026-08-25T19:36:50.700241881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fc5c0\u003e: \n failed to create HCP cluster np-downgrade-9wkwgd: failed waiting for cluster=\"np-downgrade-9wkwgd\" in resourcegroup=\"rg-np-downgrade-9wkwgd-c6w4l726nv9w\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4ede516-845b-4e65-988d-164f84fc3105\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4ede516-845b-4e65-988d-164f84fc3105\",\n \"name\": \"b4ede516-845b-4e65-988d-164f84fc3105\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.55170955Z\",\n \"endTime\": \"2026-08-25T19:36:50.700241881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"bdffa9ec41658c3c0b2fef63b1f22f6a6742e6c4e837e0fea114368b4612c51e","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"4ca0d90c5accc251f79a548ac4f235fa37708ec7a5a06deeb65a77bd99992647","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"a5e9145697fe5d26a793f74527aa00e2ec11f8e0d29fd25f899226a0c8ca6786","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","signature_id":"f337aa7b62ef749e20f60dfec4dfcb7a4238a15c3da0837817bfff818d840f30","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"77a4fccb98689590052e444032f6e0b2a81fc76d19cc8dedaba7314fbe9ef5b2","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"f3706b7b68e9a79a6664d2f4e9bb40f34471b0eda04d386d956f5c3d42464cb6","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"020633b5339d75f1df2cb64c5d4e04f6ce5d65297c8f1dc716fac948850492b0","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"50cfe241bd776c63ead0185c89db291d20bfefd14fa4462a1eff23a8bb7280f2","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"d6949fb19ef2eea4676f8abc14bdf9d0083de17b97edd1d98d989d65f25a63ef","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"841f5d6b1662eac99a6cc527de5c5a356bc8edcef21537485d177eab3f47e0b8","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"f7a0795ccf41d2868c0062e96d14bdc47a40e0b4d69128fb529e4080a9cfca2a","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"37180a3567b7ba4c19f170d6ea685b79ea6d88f1f355e96431dbe0380eee9693","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"ab0efe968c3af5d317e92a58ea9e41044d2452ac55e715d3c4270447bf41e742","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":11,"runs_affected":10,"failed_runs":10,"run_impact_percent":4.166666666666667,"anchor_occurrences":11,"after_last_push_count":0,"trend_counts":[0,0,11,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010cc300\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-j56jzgdfb84c\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/152cdec3-f7df-4add-8bd0-f0c465a1cce4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/152cdec3-f7df-4add-8bd0-f0c465a1cce4\",\n \"name\": \"152cdec3-f7df-4add-8bd0-f0c465a1cce4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:55.074048426Z\",\n \"endTime\": \"2026-08-25T19:42:02.420103182Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010cc300\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-j56jzgdfb84c\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/152cdec3-f7df-4add-8bd0-f0c465a1cce4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/152cdec3-f7df-4add-8bd0-f0c465a1cce4\",\n \"name\": \"152cdec3-f7df-4add-8bd0-f0c465a1cce4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:55.074048426Z\",\n \"endTime\": \"2026-08-25T19:42:02.420103182Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00128a380\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7p5xtj25zdgq\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02c1dd80-6bb6-452a-8773-44a463672ab0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02c1dd80-6bb6-452a-8773-44a463672ab0\",\n \"name\": \"02c1dd80-6bb6-452a-8773-44a463672ab0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:27.029566085Z\",\n \"endTime\": \"2026-08-25T19:42:33.615773728Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00128a380\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7p5xtj25zdgq\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02c1dd80-6bb6-452a-8773-44a463672ab0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02c1dd80-6bb6-452a-8773-44a463672ab0\",\n \"name\": \"02c1dd80-6bb6-452a-8773-44a463672ab0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:27.029566085Z\",\n \"endTime\": \"2026-08-25T19:42:33.615773728Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-hrhcgp\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dce540\u003e: \n failed to create HCP cluster cluster-zstream-5-0-hrhcgp: failed waiting for cluster=\"cluster-zstream-5-0-hrhcgp\" in resourcegroup=\"rg-zstream-upgrade-5-0-78mnn44hkdrc\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b95d4f1a-9b50-422c-9b91-4f716507dd2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b95d4f1a-9b50-422c-9b91-4f716507dd2d\",\n \"name\": \"b95d4f1a-9b50-422c-9b91-4f716507dd2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:27.785135797Z\",\n \"endTime\": \"2026-08-25T16:40:28.593240311Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-hrhcgp\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dce540\u003e: \n failed to create HCP cluster cluster-zstream-5-0-hrhcgp: failed waiting for cluster=\"cluster-zstream-5-0-hrhcgp\" in resourcegroup=\"rg-zstream-upgrade-5-0-78mnn44hkdrc\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b95d4f1a-9b50-422c-9b91-4f716507dd2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b95d4f1a-9b50-422c-9b91-4f716507dd2d\",\n \"name\": \"b95d4f1a-9b50-422c-9b91-4f716507dd2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:27.785135797Z\",\n \"endTime\": \"2026-08-25T16:40:28.593240311Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"e7d19e4ec13af075778836b0509e48aae2d3aa918859115e31905f9216f6eda2","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"2a3c305e2e7b889cc83bd789e19181806498ad35731017cdff9ccc00c9807367","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"04200eec52043650488776ea91f3a75dc9e4bb6cd15be64da7bc9e97ceff3c5e","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"5aee83e4a720dd3ebb6047456af4c05c8c39902dcd0a2d8df1e2007ebdf2f0c2","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"b2eebc2a537ba9b4668bce19ae91591fac3e22a5f8939d98e9175fbce444f838","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"0df35959863c4a18d18241249045541a76f7fb86ff7bc0c329f8da11e7941ea4","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"5ae8b49cfd62a1fd2be6d40eb9b8f7b2df783c47f66002e18ea549ef9fae6614","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"8264131fc2b45d2f5ca5d9936dd76c0aeb91f30f562df92f37bbc56a238e6d26","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"cfec1836d28d37c357fa4cdaaa7a7e2cfa94f1863cd3c937f9210fab49a6b769","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"b5ee1f8ad40d4305b4dfae8b6e148d13794c07d1d9f1c5fc72025bff8e810890","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"cf304d6dfff36c6441a2fb53d72be066b0745f911a7fb1567e495cd2a83bfd4e","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":10,"runs_affected":8,"failed_runs":8,"run_impact_percent":3.3333333333333335,"anchor_occurrences":10,"after_last_push_count":3,"trend_counts":[0,0,10,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":17,"prior_last_seen_at":"2026-08-21T06:13:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00131c880\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-bxjjlgfmc9db\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3a4169eb-2032-480c-8f15-d84b8528bedf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3a4169eb-2032-480c-8f15-d84b8528bedf\",\n \"name\": \"3a4169eb-2032-480c-8f15-d84b8528bedf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:19.200664013Z\",\n \"endTime\": \"2026-08-25T19:42:23.611070686Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00131c880\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-bxjjlgfmc9db\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3a4169eb-2032-480c-8f15-d84b8528bedf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3a4169eb-2032-480c-8f15-d84b8528bedf\",\n \"name\": \"3a4169eb-2032-480c-8f15-d84b8528bedf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:19.200664013Z\",\n \"endTime\": \"2026-08-25T19:42:23.611070686Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012ea320\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-h5k4cp2j9wxp\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36f15d29-956b-4c81-a94e-9ff3c38026e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36f15d29-956b-4c81-a94e-9ff3c38026e3\",\n \"name\": \"36f15d29-956b-4c81-a94e-9ff3c38026e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:46.898858572Z\",\n \"endTime\": \"2026-08-25T19:36:50.642648068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012ea320\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-h5k4cp2j9wxp\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36f15d29-956b-4c81-a94e-9ff3c38026e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36f15d29-956b-4c81-a94e-9ff3c38026e3\",\n \"name\": \"36f15d29-956b-4c81-a94e-9ff3c38026e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:46.898858572Z\",\n \"endTime\": \"2026-08-25T19:36:50.642648068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-9v2fm7\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abc760\u003e: \n failed to create HCP cluster cluster-zstream-4-22-9v2fm7: failed waiting for cluster=\"cluster-zstream-4-22-9v2fm7\" in resourcegroup=\"rg-zstream-upgrade-4-22-qzxxgjsj26qb\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02d2964a-7a05-4345-980d-51e9bdf2ffaf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02d2964a-7a05-4345-980d-51e9bdf2ffaf\",\n \"name\": \"02d2964a-7a05-4345-980d-51e9bdf2ffaf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:54:16.06655817Z\",\n \"endTime\": \"2026-08-25T15:13:19.307227089Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-9v2fm7\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abc760\u003e: \n failed to create HCP cluster cluster-zstream-4-22-9v2fm7: failed waiting for cluster=\"cluster-zstream-4-22-9v2fm7\" in resourcegroup=\"rg-zstream-upgrade-4-22-qzxxgjsj26qb\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02d2964a-7a05-4345-980d-51e9bdf2ffaf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02d2964a-7a05-4345-980d-51e9bdf2ffaf\",\n \"name\": \"02d2964a-7a05-4345-980d-51e9bdf2ffaf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:54:16.06655817Z\",\n \"endTime\": \"2026-08-25T15:13:19.307227089Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"7920aacad73a762e7cb4200cc3f935f23f19001cc15e0d371213e2d503e50bf5","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"c1c3211636089d001497e52a1029d888d6af4966bdcf5a039a78f8eb9fff0edc","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"c3b3ef09c20cc547bca5325c7f41e4dd0c4a174e6693ff2418f6a450ed448c6b","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","occurred_at":"2026-08-25T13:34:16Z","signature_id":"ba2c82cf6ae3a73e48ea720374e43d7b38d997b258802c349e2dac2f93c95ba4","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"941ca1754672c53670ae6f9235025ce03ee81e89e2ab50e5ebdc653a9104513b","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"bd627bb7dd0c1e61533ffcaff0de1677c51817a542e05bd6bac955235f59715c","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"d170f04f7cb6344abd769b118fc7f073e431f9b518426774c88ff7532f8e723d","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"036e61e55afe3d161610066648e9dacc1c80f9cbd63ca3be6cf42026860462b7","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"2a0367cc0e9683bd02dfae7b534805fb13ea99e1bfc239a750d74f7845bd9014","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"6a014dc04f005c0ce2a8189b9d8c8434c9d29f69cb03bf76fae5f1ea1b289999","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":9,"runs_affected":8,"failed_runs":8,"run_impact_percent":3.3333333333333335,"anchor_occurrences":9,"after_last_push_count":1,"trend_counts":[0,0,9,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":13,"prior_last_seen_at":"2026-08-18T18:57:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007c92e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8wz885flg8mk\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f6cc866-fb13-4391-b5e4-4ef1f14b4973\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f6cc866-fb13-4391-b5e4-4ef1f14b4973\",\n \"name\": \"9f6cc866-fb13-4391-b5e4-4ef1f14b4973\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:24:58.553705193Z\",\n \"endTime\": \"2026-08-25T19:44:03.525820627Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007c92e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8wz885flg8mk\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f6cc866-fb13-4391-b5e4-4ef1f14b4973\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f6cc866-fb13-4391-b5e4-4ef1f14b4973\",\n \"name\": \"9f6cc866-fb13-4391-b5e4-4ef1f14b4973\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:24:58.553705193Z\",\n \"endTime\": \"2026-08-25T19:44:03.525820627Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f71140\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7fhrl8nsqhmm\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\",\n \"name\": \"9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:28.727590226Z\",\n \"endTime\": \"2026-08-25T16:40:28.622162047Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f71140\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7fhrl8nsqhmm\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\",\n \"name\": \"9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:28.727590226Z\",\n \"endTime\": \"2026-08-25T16:40:28.622162047Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013f6760\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-8qxvb7djrfqs\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ef3c075-0505-498f-acb7-bec851217620\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ef3c075-0505-498f-acb7-bec851217620\",\n \"name\": \"0ef3c075-0505-498f-acb7-bec851217620\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:13:28.807201061Z\",\n \"endTime\": \"2026-08-25T16:32:29.809496877Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013f6760\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-8qxvb7djrfqs\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ef3c075-0505-498f-acb7-bec851217620\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ef3c075-0505-498f-acb7-bec851217620\",\n \"name\": \"0ef3c075-0505-498f-acb7-bec851217620\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:13:28.807201061Z\",\n \"endTime\": \"2026-08-25T16:32:29.809496877Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"acd72cf8b7bb2f45dae55eed6a696c8c2095d677ca82dcc3ccb5ef413f64290e","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"8736192f6f4f645775b5c68f46d7ee7ddc0264e06e6677e85bab2a90cd28f2bc","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","signature_id":"d90c9eedcebd8ebee811c56262835a367e830ea3ef985a6b9e0519703d7c4512","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"bccfe75fe11d0ea8f7728479404e3d27d3b6a70c50e8e5d0793d665e9d222fbf","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"01b87695dd9b2116a68fd8b6e0c91abd3e780e11edb0177bfb654801580cf72d","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"1f29965bd436288d269b00ceb20f18dc47e327d03580c8db54436fbf801d2664","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"6aa2e7ee830b9fa0230c736bace065dc0ed283e1350606e8f5015746076bf57f","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"ddfd07ad7c5094cf706dd735ad3c3aed8ba35bd4d8f19d89a430d7152ef2a389","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"bbe4d66acf643736fa2f7c6c58f4e7114799ee91a201d8f3333899113347be58","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":3.3333333333333335,"anchor_occurrences":8,"after_last_push_count":10,"trend_counts":[0,3,5,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":87,"prior_last_seen_at":"2026-08-21T20:12:08Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":8}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T17:40:35Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j0166528-svc\", component=\"backend\", name=\"triggercontrolplaneupgrade\", severity=\"warning\"\n Description: Backend controller workqueue triggercontrolplaneupgrade has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T16:39:54Z\n Ended: 2026-08-25T17:06:55Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j0912128-svc\", component=\"backend\", name=\"clusterresourcescontroller\", severity=\"warning\"\n Description: Backend controller workqueue clusterresourcescontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T11:10:19Z\n Ended: 2026-08-25T11:46:21Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j8697984-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"417c675e722a3d2e9207698265ffad01510d51821854d76bfab92157497f8c02","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146691078623232","occurred_at":"2026-08-25T07:06:55Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071627658366976","occurred_at":"2026-08-25T02:08:39Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092031971118026752","occurred_at":"2026-08-24T23:30:58Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091995273302642688","occurred_at":"2026-08-24T21:05:15Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":3.3333333333333335,"anchor_occurrences":8,"after_last_push_count":0,"trend_counts":[0,2,6,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-22T00:32:00Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":8}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T19:49:51Z\n Ended: 2026-08-25T19:57:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8405376-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8405376-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T19:50:28Z\n Ended: 2026-08-25T19:56:27Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7285632-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7285632-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T19:13:57Z\n Ended: 2026-08-25T19:41:56Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6140160-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6140160-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"57c5ec60d7b5f08bc41aec5dff584edf947ed9bf0d23769b021eb635a33b3d21","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"0ca8bc99c3c40c8ea160690445d9fbca4420866493398175f96a3e330d91ed47","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092313647266140160","occurred_at":"2026-08-25T18:10:21Z","signature_id":"42ec01f8964f22f9352e4ea60e7c8934ff52d4ff3ff5e12af66623897e7c4751","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261270257405952","occurred_at":"2026-08-25T14:42:10Z","signature_id":"a433ce52553f210f473828de792bdb37ba8e12f31a35d7427eb780d885009233","pr_number":6614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"f9b1dbaf8f8300b20f00ab73fece95cb317fd0f48d3a396dbbe2c51c020ad097","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092188089500831744","occurred_at":"2026-08-25T09:29:48Z","signature_id":"6db69bc2fe95a94bfd8232f25627f1581b6967e549eb3db637637bd4cef65f38","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932359686361088","occurred_at":"2026-08-24T16:54:34Z","signature_id":"a3d3dfe16c45487c7de4833dcff7c72d504e34a1b391992ceada6266956233ad","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091921367619342336","occurred_at":"2026-08-24T16:10:41Z","signature_id":"d4e52f6e7208e4a023c43c3ea9a95264341c256a3dcba5d703516c2b2c81f97b","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":8,"runs_affected":5,"failed_runs":5,"run_impact_percent":2.0833333333333335,"anchor_occurrences":8,"after_last_push_count":0,"trend_counts":[0,2,6,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86580\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qvdpwsdrhf5d\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/11ae9dbb-7a03-4823-a826-d4717833e908\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/11ae9dbb-7a03-4823-a826-d4717833e908\",\n \"name\": \"11ae9dbb-7a03-4823-a826-d4717833e908\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:13:50.746895077Z\",\n \"endTime\": \"2026-08-25T16:32:59.846114727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86580\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qvdpwsdrhf5d\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/11ae9dbb-7a03-4823-a826-d4717833e908\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/11ae9dbb-7a03-4823-a826-d4717833e908\",\n \"name\": \"11ae9dbb-7a03-4823-a826-d4717833e908\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:13:50.746895077Z\",\n \"endTime\": \"2026-08-25T16:32:59.846114727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-x6bg4m6clbjq/cluster-candidate-4-22-v8dh2g should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001724ba0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-v8dh2g: failed waiting for cluster=\"cluster-candidate-4-22-v8dh2g\" in resourcegroup=\"rg-candidate-4-22-x6bg4m6clbjq\" to finish creating: GET https://rp.j8970496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b6be2ed-087d-4652-a133-bf375421a904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b6be2ed-087d-4652-a133-bf375421a904\",\n \"name\": \"8b6be2ed-087d-4652-a133-bf375421a904\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:17.286340435Z\",\n \"endTime\": \"2026-08-25T14:59:19.220463305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-x6bg4m6clbjq/cluster-candidate-4-22-v8dh2g should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001724ba0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-v8dh2g: failed waiting for cluster=\"cluster-candidate-4-22-v8dh2g\" in resourcegroup=\"rg-candidate-4-22-x6bg4m6clbjq\" to finish creating: GET https://rp.j8970496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b6be2ed-087d-4652-a133-bf375421a904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b6be2ed-087d-4652-a133-bf375421a904\",\n \"name\": \"8b6be2ed-087d-4652-a133-bf375421a904\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:17.286340435Z\",\n \"endTime\": \"2026-08-25T14:59:19.220463305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da43a0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-52lwzkdvrvxq\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1c5237ca-aea6-4af4-af59-1bac03bfc222\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1c5237ca-aea6-4af4-af59-1bac03bfc222\",\n \"name\": \"1c5237ca-aea6-4af4-af59-1bac03bfc222\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:09.556002909Z\",\n \"endTime\": \"2026-08-25T14:42:14.793443561Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da43a0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-52lwzkdvrvxq\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1c5237ca-aea6-4af4-af59-1bac03bfc222\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1c5237ca-aea6-4af4-af59-1bac03bfc222\",\n \"name\": \"1c5237ca-aea6-4af4-af59-1bac03bfc222\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:09.556002909Z\",\n \"endTime\": \"2026-08-25T14:42:14.793443561Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","signature_id":"4bc11107f133f9177edbdf6e9f119a737521511f6edc5e691bc1878c9ba1f527","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","signature_id":"9de0b947dcfc7f945c30a376f6c6b3a6af42020e652c40b79da8b091bb68bf00","pr_number":6677,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"2f9c966df057075e45f28d9a86360c4da74e7efb2d31b2a981af2a97722900a4","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"dabbb7f9069ce3251099eefa51315c3d0e7f90c58175bd7bad32d9f2f8d81b40","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"e8bbe3fd18272cae3ce8b375d491cf16615e129bb95cc62f5766586a6be8de21","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"9fed5947fac9eb6a4be4cefb16ab0947113474d3eee1eb30b287e0d183316002","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"ab3b51419f8536317834b49d26ead032c3d2df538686a40306860f09479fc7a4","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"fb242b6b09da99dc7298c2563fdd22c18c2c33f65971895aaf866ce3967b98d3","pr_number":6641,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":2.9166666666666665,"anchor_occurrences":7,"after_last_push_count":1,"trend_counts":[0,1,6,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":29,"prior_last_seen_at":"2026-08-20T22:52:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6658,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","failure_pattern":"alert [svc] UJClusterProvisionErrors6h30m fired","search_query":"alert [svc] UJClusterProvisionErrors6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":2.9166666666666665,"anchor_occurrences":7,"after_last_push_count":2,"trend_counts":[0,0,7,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":27,"prior_last_seen_at":"2026-08-21T01:50:52Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","occurrences":7}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T17:11:35Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j0672512-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0672512/providers/microsoft.monitor/accounts/services-j0672512\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T15:14:00Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j9549056-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9549056/providers/microsoft.monitor/accounts/services-j9549056\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T12:06:25Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j9005312-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9005312/providers/microsoft.monitor/accounts/services-j9005312\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"23aa9d7dee7c9e529060801e9167a47fb7a497985cb1d61e9db28b65884ea401","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"38c81dc69b06278ed233e0939f35e8c71cf8eff2f9cb495db24d74f240d5e1a1","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"af8dcfdf2cbedd9372f04cbd90abd56a127bdc4079bafb1ef1913edb6177347d","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"8e7a90f4497ef1b51e90f2043b6517b8c68d8bfd66277c1d9bf290617a22ad90","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"77f2e0ebb4f832bfccfdc5298c2a752e61abc579b540c5ba9eb248bda7ab6b07","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"c4c245d9532eb880b4ccd3795a4e4bdc7b51cc2471678008cfc9c24b079969f8","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"27c12cddda1a8af9ca8f43e94a34fd689832822c5edc234c8b00528b723b3ae1","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":7,"runs_affected":6,"failed_runs":6,"run_impact_percent":2.5,"anchor_occurrences":7,"after_last_push_count":0,"trend_counts":[0,1,6,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-22T18:55:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cce280\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rg4h7xfrjp8d\" to finish creating: GET https://rp.j4940032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e2c6a53-377f-4d0e-9bd7-168eb63a2802\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e2c6a53-377f-4d0e-9bd7-168eb63a2802\",\n \"name\": \"9e2c6a53-377f-4d0e-9bd7-168eb63a2802\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:24:41.72360651Z\",\n \"endTime\": \"2026-08-25T15:43:49.738296088Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-version-operator, cluster-storage-operator, ingress-operator, hosted-cluster-config-operator, packageserver, cluster-node-tuning-operator, olm-operator, redhat-operators-catalog, community-operators-catalog, dns-operator, catalog-operator, certified-operators-catalog, cluster-policy-controller, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cce280\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rg4h7xfrjp8d\" to finish creating: GET https://rp.j4940032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e2c6a53-377f-4d0e-9bd7-168eb63a2802\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e2c6a53-377f-4d0e-9bd7-168eb63a2802\",\n \"name\": \"9e2c6a53-377f-4d0e-9bd7-168eb63a2802\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:24:41.72360651Z\",\n \"endTime\": \"2026-08-25T15:43:49.738296088Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-version-operator, cluster-storage-operator, ingress-operator, hosted-cluster-config-operator, packageserver, cluster-node-tuning-operator, olm-operator, redhat-operators-catalog, community-operators-catalog, dns-operator, catalog-operator, certified-operators-catalog, cluster-policy-controller, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-pgpvxwpm6mjj/cluster-candidate-4-22-bnptgd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084220\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bnptgd: failed waiting for cluster=\"cluster-candidate-4-22-bnptgd\" in resourcegroup=\"rg-candidate-4-22-pgpvxwpm6mjj\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b420dfc-8b86-46f6-8ff3-d7254009e5e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b420dfc-8b86-46f6-8ff3-d7254009e5e3\",\n \"name\": \"9b420dfc-8b86-46f6-8ff3-d7254009e5e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.515128304Z\",\n \"endTime\": \"2026-08-25T11:24:07.660901963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, community-operators-catalog, cluster-node-tuning-operator, catalog-operator, certified-operators-catalog, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, cluster-network-operator, dns-operator, ingress-operator, packageserver, openshift-controller-manager, cluster-storage-operator, olm-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-pgpvxwpm6mjj/cluster-candidate-4-22-bnptgd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084220\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bnptgd: failed waiting for cluster=\"cluster-candidate-4-22-bnptgd\" in resourcegroup=\"rg-candidate-4-22-pgpvxwpm6mjj\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b420dfc-8b86-46f6-8ff3-d7254009e5e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b420dfc-8b86-46f6-8ff3-d7254009e5e3\",\n \"name\": \"9b420dfc-8b86-46f6-8ff3-d7254009e5e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.515128304Z\",\n \"endTime\": \"2026-08-25T11:24:07.660901963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, community-operators-catalog, cluster-node-tuning-operator, catalog-operator, certified-operators-catalog, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, cluster-network-operator, dns-operator, ingress-operator, packageserver, openshift-controller-manager, cluster-storage-operator, olm-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f761c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-tsn8fjg7z8v8\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c60a9b4-98e8-437a-b0d1-340131fafe1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c60a9b4-98e8-437a-b0d1-340131fafe1e\",\n \"name\": \"7c60a9b4-98e8-437a-b0d1-340131fafe1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.548851412Z\",\n \"endTime\": \"2026-08-25T11:21:35.946063021Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, cluster-version-operator, openshift-route-controller-manager, olm-operator, packageserver, certified-operators-catalog, cluster-node-tuning-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, ingress-operator, redhat-operators-catalog, openshift-controller-manager, community-operators-catalog, hosted-cluster-config-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f761c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-tsn8fjg7z8v8\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c60a9b4-98e8-437a-b0d1-340131fafe1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c60a9b4-98e8-437a-b0d1-340131fafe1e\",\n \"name\": \"7c60a9b4-98e8-437a-b0d1-340131fafe1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.548851412Z\",\n \"endTime\": \"2026-08-25T11:21:35.946063021Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, cluster-version-operator, openshift-route-controller-manager, olm-operator, packageserver, certified-operators-catalog, cluster-node-tuning-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, ingress-operator, redhat-operators-catalog, openshift-controller-manager, community-operators-catalog, hosted-cluster-config-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092257519584940032","occurred_at":"2026-08-25T14:27:19Z","signature_id":"1420b0f2188f90f92326018bb88dfc5b387aae62ec7dcbd9fa65850c57c8d47e","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"1f069fa0311578db887619d6e5fa3d28f357906bf03f02fdb94770dedff8043b","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"7e3686375be61c20eb803573e751563f0941ff5159af2cf0a463b90d62b70143","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"b6335e195fbae35825fcde7af066eb7de6c819b9fcc823ca2c6c99c2440ef185","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"1cb8534c4d4dcf75a5e26bdcd0fd6bf39f4b3093ae4d3e32c0a5e2dd8fbdb573","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"946e42b1ec8ff2dfdb875eabb2e23ed3cccd0fb9157427c09f12a2be51c45b2b","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"6b56df09ac5297e89b7853c7486e42c460930ae4b037271fddfef92b6ce0846a","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":7,"runs_affected":6,"failed_runs":6,"run_impact_percent":2.5,"anchor_occurrences":7,"after_last_push_count":0,"trend_counts":[0,0,7,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005e2b40\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-gvqsnvnk8pkx\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60298cc3-cc25-45ff-b759-a7365c10ec56\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60298cc3-cc25-45ff-b759-a7365c10ec56\",\n \"name\": \"60298cc3-cc25-45ff-b759-a7365c10ec56\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:21.389463971Z\",\n \"endTime\": \"2026-08-25T19:39:22.423838537Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005e2b40\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-gvqsnvnk8pkx\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60298cc3-cc25-45ff-b759-a7365c10ec56\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60298cc3-cc25-45ff-b759-a7365c10ec56\",\n \"name\": \"60298cc3-cc25-45ff-b759-a7365c10ec56\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:21.389463971Z\",\n \"endTime\": \"2026-08-25T19:39:22.423838537Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019daa40\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-g5pwkrb85fq6\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a089fdb6-b47d-4203-b58e-db6f9734d598\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a089fdb6-b47d-4203-b58e-db6f9734d598\",\n \"name\": \"a089fdb6-b47d-4203-b58e-db6f9734d598\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:12.44361008Z\",\n \"endTime\": \"2026-08-25T14:40:13.632066828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019daa40\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-g5pwkrb85fq6\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a089fdb6-b47d-4203-b58e-db6f9734d598\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a089fdb6-b47d-4203-b58e-db6f9734d598\",\n \"name\": \"a089fdb6-b47d-4203-b58e-db6f9734d598\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:12.44361008Z\",\n \"endTime\": \"2026-08-25T14:40:13.632066828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-5vxfzn with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005eed00\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-5vxfzn: failed waiting for cluster=\"np-version-upgrade-cluster-5vxfzn\" in resourcegroup=\"rg-np-version-upgrade-5vxfzn-7svm57fp7wk7\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fa03eeb-ac17-4494-af34-d2fa27ce5596\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fa03eeb-ac17-4494-af34-d2fa27ce5596\",\n \"name\": \"1fa03eeb-ac17-4494-af34-d2fa27ce5596\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:54.133785755Z\",\n \"endTime\": \"2026-08-25T11:20:57.201457228Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, olm-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-5vxfzn with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005eed00\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-5vxfzn: failed waiting for cluster=\"np-version-upgrade-cluster-5vxfzn\" in resourcegroup=\"rg-np-version-upgrade-5vxfzn-7svm57fp7wk7\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fa03eeb-ac17-4494-af34-d2fa27ce5596\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fa03eeb-ac17-4494-af34-d2fa27ce5596\",\n \"name\": \"1fa03eeb-ac17-4494-af34-d2fa27ce5596\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:54.133785755Z\",\n \"endTime\": \"2026-08-25T11:20:57.201457228Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, olm-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"383f94fc81a983b134e7f8347429995df9c16fd5e3b4d726b4e05d385fc27709","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"a9c57aba5f42be110377a5cf9ef9c173f8bee026a837d5411882de02fec7d3cc","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"effe32de00004a904c211837be3300222c5b0de9f1d089f9996c139580d10677","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"aabd7b54cdc1ced5ff8794a6e08fcc1a9d50f3d188a6e28369345aad73a983a2","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"2481cd9dd55cb11b365e311cdeb7a7671bbaebdfc0f19a6ea5043f7d815bc754","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"5d116f9528929d7006556d8f84e31a0f89f4b8bd4db46d055fca3f0978d9a496","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"6e775195179703ce3594578f28d014a111b897b75498606daa63da0196f2a8f5","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":7,"runs_affected":6,"failed_runs":6,"run_impact_percent":2.5,"anchor_occurrences":7,"after_last_push_count":0,"trend_counts":[0,1,6,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee8180\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rnmt78kw8wzg\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae74a4ae-565f-4545-9b82-af68f6c7938b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae74a4ae-565f-4545-9b82-af68f6c7938b\",\n \"name\": \"ae74a4ae-565f-4545-9b82-af68f6c7938b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:28.849719343Z\",\n \"endTime\": \"2026-08-25T19:38:30.742891601Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, openshift-controller-manager, azure-cloud-controller-manager, cluster-image-registry-operator, olm-operator, cluster-version-operator, ignition-server, redhat-operators-catalog, community-operators-catalog, control-plane-pki-operator, cluster-policy-controller, cluster-storage-operator, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, catalog-operator, cluster-node-tuning-operator, ignition-server-proxy, ingress-operator, olm-collect-profiles, certified-operators-catalog, machine-approver, kube-controller-manager, dns-operator, konnectivity-agent, redhat-marketplace-catalog, cluster-autoscaler, openshift-route-controller-manager, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee8180\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rnmt78kw8wzg\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae74a4ae-565f-4545-9b82-af68f6c7938b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae74a4ae-565f-4545-9b82-af68f6c7938b\",\n \"name\": \"ae74a4ae-565f-4545-9b82-af68f6c7938b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:28.849719343Z\",\n \"endTime\": \"2026-08-25T19:38:30.742891601Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, openshift-controller-manager, azure-cloud-controller-manager, cluster-image-registry-operator, olm-operator, cluster-version-operator, ignition-server, redhat-operators-catalog, community-operators-catalog, control-plane-pki-operator, cluster-policy-controller, cluster-storage-operator, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, catalog-operator, cluster-node-tuning-operator, ignition-server-proxy, ingress-operator, olm-collect-profiles, certified-operators-catalog, machine-approver, kube-controller-manager, dns-operator, konnectivity-agent, redhat-marketplace-catalog, cluster-autoscaler, openshift-route-controller-manager, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000942ca0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kzttj9k8trpk\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/848dc16e-8d2f-492e-ad78-0e05ac46ed06\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/848dc16e-8d2f-492e-ad78-0e05ac46ed06\",\n \"name\": \"848dc16e-8d2f-492e-ad78-0e05ac46ed06\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:22:26.59789954Z\",\n \"endTime\": \"2026-08-25T16:41:28.524110603Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, ignition-server-proxy, cluster-policy-controller, hosted-cluster-config-operator, olm-collect-profiles, redhat-marketplace-catalog, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, cluster-node-tuning-operator, ingress-operator, packageserver, openshift-apiserver, cluster-image-registry-operator, olm-operator, openshift-controller-manager, catalog-operator, community-operators-catalog, ignition-server, redhat-operators-catalog, konnectivity-agent, kube-scheduler, azure-cloud-controller-manager, certified-operators-catalog, machine-approver, cluster-storage-operator, dns-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000942ca0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kzttj9k8trpk\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/848dc16e-8d2f-492e-ad78-0e05ac46ed06\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/848dc16e-8d2f-492e-ad78-0e05ac46ed06\",\n \"name\": \"848dc16e-8d2f-492e-ad78-0e05ac46ed06\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:22:26.59789954Z\",\n \"endTime\": \"2026-08-25T16:41:28.524110603Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, ignition-server-proxy, cluster-policy-controller, hosted-cluster-config-operator, olm-collect-profiles, redhat-marketplace-catalog, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, cluster-node-tuning-operator, ingress-operator, packageserver, openshift-apiserver, cluster-image-registry-operator, olm-operator, openshift-controller-manager, catalog-operator, community-operators-catalog, ignition-server, redhat-operators-catalog, konnectivity-agent, kube-scheduler, azure-cloud-controller-manager, certified-operators-catalog, machine-approver, cluster-storage-operator, dns-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014d4ba0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-djpx8vzhvdqj\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a29c2b48-aa8b-4185-a975-2ba36d80abfa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a29c2b48-aa8b-4185-a975-2ba36d80abfa\",\n \"name\": \"a29c2b48-aa8b-4185-a975-2ba36d80abfa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:32.971395174Z\",\n \"endTime\": \"2026-08-25T11:20:37.411972437Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, control-plane-pki-operator, cluster-storage-operator, dns-operator, ingress-operator, ignition-server, olm-collect-profiles, certified-operators-catalog, kube-controller-manager, openshift-apiserver, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-network-operator, konnectivity-agent, ignition-server-proxy, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, cluster-autoscaler, machine-approver, packageserver, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014d4ba0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-djpx8vzhvdqj\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a29c2b48-aa8b-4185-a975-2ba36d80abfa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a29c2b48-aa8b-4185-a975-2ba36d80abfa\",\n \"name\": \"a29c2b48-aa8b-4185-a975-2ba36d80abfa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:32.971395174Z\",\n \"endTime\": \"2026-08-25T11:20:37.411972437Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, control-plane-pki-operator, cluster-storage-operator, dns-operator, ingress-operator, ignition-server, olm-collect-profiles, certified-operators-catalog, kube-controller-manager, openshift-apiserver, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-network-operator, konnectivity-agent, ignition-server-proxy, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, cluster-autoscaler, machine-approver, packageserver, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"24ef7c09d5d3ddd04909d1a88daf40487983c7d0988d0358607e410a3fcbd326","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"723d424b638d374276f7f3771ede71c77d1aeb31880363605a2c07414b4ed15d","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"9a6939f0032bf8fdc09bb65ca1155a9ec01d12b02a56acd5a25adfca6beb61f3","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"9c1b8b2a2c406c20c5278d4329b8f1b42ee01e05e65a61c633a54cbe6a5372e5","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"17d508ac7b58e43e34e867e2daa42e985d102b6efa1571d29f852cfe4e849659","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"bac68d05e70928d803a2e96e39a3594853024ebe374dbc961bb7717d91c26078","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"14a501f15a848fe949827032cbe7eab1732753bec28bb5cacc0d3876ab277f28","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"03e76cdaabc445ed66f9db895033ab5c05ca755861d993eb55fd2ea99afdd76d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":6,"runs_affected":4,"failed_runs":4,"run_impact_percent":1.6666666666666667,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[0,0,6,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af62e0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5j68lm294d7v\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4433c13-3a16-45ba-88e7-ec043217acd4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4433c13-3a16-45ba-88e7-ec043217acd4\",\n \"name\": \"e4433c13-3a16-45ba-88e7-ec043217acd4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:40.830511566Z\",\n \"endTime\": \"2026-08-25T11:20:48.170023371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af62e0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5j68lm294d7v\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4433c13-3a16-45ba-88e7-ec043217acd4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4433c13-3a16-45ba-88e7-ec043217acd4\",\n \"name\": \"e4433c13-3a16-45ba-88e7-ec043217acd4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:40.830511566Z\",\n \"endTime\": \"2026-08-25T11:20:48.170023371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-bp8trx\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d85000\u003e: \n failed to create HCP cluster cluster-zstream-5-0-bp8trx: failed waiting for cluster=\"cluster-zstream-5-0-bp8trx\" in resourcegroup=\"rg-zstream-upgrade-5-0-brkg7nwpr7jw\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b253226-385a-4503-a738-5e57c00054ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b253226-385a-4503-a738-5e57c00054ba\",\n \"name\": \"3b253226-385a-4503-a738-5e57c00054ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:19.290629965Z\",\n \"endTime\": \"2026-08-25T11:20:27.735529124Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-bp8trx\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d85000\u003e: \n failed to create HCP cluster cluster-zstream-5-0-bp8trx: failed waiting for cluster=\"cluster-zstream-5-0-bp8trx\" in resourcegroup=\"rg-zstream-upgrade-5-0-brkg7nwpr7jw\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b253226-385a-4503-a738-5e57c00054ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b253226-385a-4503-a738-5e57c00054ba\",\n \"name\": \"3b253226-385a-4503-a738-5e57c00054ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:19.290629965Z\",\n \"endTime\": \"2026-08-25T11:20:27.735529124Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012005e0\u003e: \n failed to create HCP cluster np-downgrade-sxtp4r: failed waiting for cluster=\"np-downgrade-sxtp4r\" in resourcegroup=\"rg-np-downgrade-sxtp4r-fhklq2vsgmt4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34134851-0102-4d35-97b3-944ff4b7f92e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34134851-0102-4d35-97b3-944ff4b7f92e\",\n \"name\": \"34134851-0102-4d35-97b3-944ff4b7f92e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:47.779072949Z\",\n \"endTime\": \"2026-08-25T11:23:47.559079763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012005e0\u003e: \n failed to create HCP cluster np-downgrade-sxtp4r: failed waiting for cluster=\"np-downgrade-sxtp4r\" in resourcegroup=\"rg-np-downgrade-sxtp4r-fhklq2vsgmt4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34134851-0102-4d35-97b3-944ff4b7f92e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34134851-0102-4d35-97b3-944ff4b7f92e\",\n \"name\": \"34134851-0102-4d35-97b3-944ff4b7f92e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:47.779072949Z\",\n \"endTime\": \"2026-08-25T11:23:47.559079763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"2634ba4314b4b33fc2951e3d9e981c753c0c9fdc75af1b8135f694c4402837b6","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"6586291399f5b3fd7dbb07efd25721a7c2283129efed74cec97d843274ba632c","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"fe41252ce6ed75f9abb776a92d3943f65f76fc05052906f4fda9f43fe1b11ce7","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"9228c5a0348f43b2f29683a574e74aac88f34daeeda42ce48207d937f6f8a3c7","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"c3a84dd7ddae67604fc665cb0bbd0f847f461d2effc89e6c2b9a218c130289a2","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"edd0549c792e30ee83baa44f1b97d25996781b695fc96efc50800b1d38ed59e2","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":6,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":6,"after_last_push_count":3,"trend_counts":[0,0,6,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-21T15:05:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001b203f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-6lrz4w\" in resourcegroup=\"rg-zstream-upgrade-4-22-7w28nhsm4n9m\" to finish deleting: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/59b20855-f596-411e-b2b5-80ac6377cccc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/59b20855-f596-411e-b2b5-80ac6377cccc\",\n \"name\": \"59b20855-f596-411e-b2b5-80ac6377cccc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:49:05.072511358Z\",\n \"endTime\": \"2026-08-25T21:13:08.479732985Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdqn1uavtmu7h5ko7gigjtjhavtoo5s still exists (deletion dispatched at 2026-08-25T20:49:06Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001b203f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-6lrz4w\" in resourcegroup=\"rg-zstream-upgrade-4-22-7w28nhsm4n9m\" to finish deleting: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/59b20855-f596-411e-b2b5-80ac6377cccc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/59b20855-f596-411e-b2b5-80ac6377cccc\",\n \"name\": \"59b20855-f596-411e-b2b5-80ac6377cccc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:49:05.072511358Z\",\n \"endTime\": \"2026-08-25T21:13:08.479732985Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdqn1uavtmu7h5ko7gigjtjhavtoo5s still exists (deletion dispatched at 2026-08-25T20:49:06Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000e7a498\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-xcmt5t2qf6ts\" to finish deleting: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f551add0-678d-420d-9d4f-3323e96b22a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f551add0-678d-420d-9d4f-3323e96b22a3\",\n \"name\": \"f551add0-678d-420d-9d4f-3323e96b22a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:32:58.476242776Z\",\n \"endTime\": \"2026-08-25T20:56:58.482834797Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdqn1ssrvb7s3s0diuquuk2e3v82jd6 still exists (deletion dispatched at 2026-08-25T20:33:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000e7a498\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-xcmt5t2qf6ts\" to finish deleting: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f551add0-678d-420d-9d4f-3323e96b22a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f551add0-678d-420d-9d4f-3323e96b22a3\",\n \"name\": \"f551add0-678d-420d-9d4f-3323e96b22a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:32:58.476242776Z\",\n \"endTime\": \"2026-08-25T20:56:58.482834797Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdqn1ssrvb7s3s0diuquuk2e3v82jd6 still exists (deletion dispatched at 2026-08-25T20:33:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0019ac228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-lhs7stnz7jdx\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee794509-c973-4d3d-9aac-193a29f17172\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee794509-c973-4d3d-9aac-193a29f17172\",\n \"name\": \"ee794509-c973-4d3d-9aac-193a29f17172\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:40:54.657536996Z\",\n \"endTime\": \"2026-08-25T17:04:58.539200774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn70pm4stdfpd62ob77dhptdq67h82 still exists (deletion dispatched at 2026-08-25T16:40:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0019ac228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-lhs7stnz7jdx\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee794509-c973-4d3d-9aac-193a29f17172\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee794509-c973-4d3d-9aac-193a29f17172\",\n \"name\": \"ee794509-c973-4d3d-9aac-193a29f17172\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:40:54.657536996Z\",\n \"endTime\": \"2026-08-25T17:04:58.539200774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn70pm4stdfpd62ob77dhptdq67h82 still exists (deletion dispatched at 2026-08-25T16:40:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092336210943938560","occurred_at":"2026-08-25T19:39:54Z","signature_id":"ce4c6493c490585b0cd4717856f3be2a8237a8d362ea4dde3067a7d9c3fa9f74","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092336210943938560","occurred_at":"2026-08-25T19:39:54Z","signature_id":"fc0ae3a9396e916198cdf07038a2582239d90214043c243fa0f905000fd68c22","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"83f994f356f863bb37e48c00b2ef945191b4a6e47aa90ca4f068fad7c6f09ca9","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"87e01991e35060c62797d52e4e26918b818f17bdf40e7387d9f553c0a433cb0e","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"9c29cfd69e7610a41eb87b6317c7a73f3a7a8990c6ce0206c424b69b0ec00888","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"fcc9fdbef9bad418f67e51fbdb9dd4579ca43684aae6fa975e7587a95e591f82","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":6,"after_last_push_count":9,"trend_counts":[0,0,6,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-14T20:39:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:137]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-wtgj75\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00152e100\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-22-wtgj75-5swmlf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-p5qqvrtstbvw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-22-p5qqvrtstbvw/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-22-wtgj75-5swmlf/operationStatuses/08584139797161101429\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-vcuqi2ogq2kwk' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:137]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-wtgj75\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00152e100\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-22-wtgj75-5swmlf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-p5qqvrtstbvw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-22-p5qqvrtstbvw/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-22-wtgj75-5swmlf/operationStatuses/08584139797161101429\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-vcuqi2ogq2kwk' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000839700\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-np-labels-taints-qdpn79\" in resourcegroup=\"rg-np-labels-taints-5t4klzqbfs75\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-5t4klzqbfs75/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-qdpn79/operationStatuses/08584139797158851285\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-eajlprfd56pyu' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000839700\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-np-labels-taints-qdpn79\" in resourcegroup=\"rg-np-labels-taints-5t4klzqbfs75\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-5t4klzqbfs75/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-qdpn79/operationStatuses/08584139797158851285\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-eajlprfd56pyu' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00184e0e0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-listd9kcp5-zjrzlw\" in resourcegroup=\"ea-list-db5549k8k7r7\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/ea-list-db5549k8k7r7/providers/Microsoft.Resources/deployments/customer-infra-ea-listd9kcp5-zjrzlw/operationStatuses/08584139797158020904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-yygnj2pbjetam' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00184e0e0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-listd9kcp5-zjrzlw\" in resourcegroup=\"ea-list-db5549k8k7r7\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/ea-list-db5549k8k7r7/providers/Microsoft.Resources/deployments/customer-infra-ea-listd9kcp5-zjrzlw/operationStatuses/08584139797158020904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-yygnj2pbjetam' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"5c83e4b12f76857c14b38ba7e2400380e5142f913e512f879520f40194148815","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"6798ec01d31b1a13a551f0d58e2e35ad1b3449b1679cb8d8c473dee35aed6098","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"af625ee5896ee6ce49c9e654442a846928a680f63a96ce2b39b4c1040f2836c7","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"b2816cbf83c46fd984a6ad136b3252029110012c9e6f9c0547ab7a88bcac4f9a","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"e607efd27bdf5f0f02b0b2627c8fcd0b747075950cb39ab6c422f0ffaddbcc9f","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"eef4016eff4f328bc4e6b8d0c0a4efcaa132ceba7408e01c03a4f07e211176c2","pr_number":6666,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":2.0833333333333335,"anchor_occurrences":5,"after_last_push_count":10,"trend_counts":[0,2,3,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":30,"prior_last_seen_at":"2026-08-21T16:05:36Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","occurrences":3},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=2)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/management/output (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-08-25T18:56:45.688Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 description=\"Step output\\n Kind: ARM\\n Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-25T18:56:46.086Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1\ntime=2026-08-25T18:56:46.474Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 deployment=74b140f8b533b36c1cdb52a5204bd298a1350ffd05ecac7c751c39ee014fab07 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3363840-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F74b140f8b533b36c1cdb52a5204bd298a1350ffd05ecac7c751c39ee014fab07\ntime=2026-08-25T18:56:46.527Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3363840-mgmt-1/providers/Microsoft.Resources/deployments/74b140f8b533b36c1cdb52a5204bd298a1350ffd05ecac7c751c39ee014fab07/operationStatuses/08584139226791753333\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment '74b140f8b533b36c1cdb52a5204bd298a1350ffd05ecac7c751c39ee014fab07' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-25T15:21:22.056Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 description=\"Step output\\n Kind: ARM\\n Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-25T15:21:22.495Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1\ntime=2026-08-25T15:21:22.967Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 deployment=5e98ce680849833c628986a2e68efe7c2667cb97ae72f93416c33a4c5e66f0ac portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6676864-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5e98ce680849833c628986a2e68efe7c2667cb97ae72f93416c33a4c5e66f0ac\ntime=2026-08-25T15:21:23.080Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6676864-mgmt-1/providers/Microsoft.Resources/deployments/5e98ce680849833c628986a2e68efe7c2667cb97ae72f93416c33a4c5e66f0ac/operationStatuses/08584139356027184199\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment '5e98ce680849833c628986a2e68efe7c2667cb97ae72f93416c33a4c5e66f0ac' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-25T14:41:26.287Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=2 description=\"Step output\\n Kind: ARM\\n Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-25T14:41:26.667Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=2\ntime=2026-08-25T14:41:27.045Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=2 deployment=3b44e853e3fa2f5ba695fe1eeadf1d38b957019eefef20ca1a49aa6b0c7a6f2b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6436096-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3b44e853e3fa2f5ba695fe1eeadf1d38b957019eefef20ca1a49aa6b0c7a6f2b\ntime=2026-08-25T14:41:27.116Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6436096-mgmt-2/providers/Microsoft.Resources/deployments/3b44e853e3fa2f5ba695fe1eeadf1d38b957019eefef20ca1a49aa6b0c7a6f2b/operationStatuses/08584139379985620464\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment '3b44e853e3fa2f5ba695fe1eeadf1d38b957019eefef20ca1a49aa6b0c7a6f2b' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314609523363840","occurred_at":"2026-08-25T18:14:10Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260182896676864","occurred_at":"2026-08-25T14:37:54Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253020946436096","occurred_at":"2026-08-25T14:09:13Z","signature_id":"3b416c8626d510d6d228f2b4d85a59b118ef1a435941087523bd321e4dac7444","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091954836298797056","occurred_at":"2026-08-24T18:15:46Z","signature_id":"0ec7efd378b135d7fa85cb93bcc2f610fe36f768df0ac4e2e0fe6122f4092f85","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928527292076032","occurred_at":"2026-08-24T16:32:12Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":4566,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":2.0833333333333335,"anchor_occurrences":5,"after_last_push_count":1,"trend_counts":[0,0,5,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-22T00:25:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-fnczl99xh8kl/cluster-candidate-4-21-jscsmk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097d140\u003e: \n failed to create HCP cluster cluster-candidate-4-21-jscsmk: failed waiting for cluster=\"cluster-candidate-4-21-jscsmk\" in resourcegroup=\"rg-candidate-4-21-fnczl99xh8kl\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c896c3fa-e9be-48ff-abf7-d53fe9674288\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c896c3fa-e9be-48ff-abf7-d53fe9674288\",\n \"name\": \"c896c3fa-e9be-48ff-abf7-d53fe9674288\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:27.440763508Z\",\n \"endTime\": \"2026-08-25T16:40:28.603771658Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-fnczl99xh8kl/cluster-candidate-4-21-jscsmk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097d140\u003e: \n failed to create HCP cluster cluster-candidate-4-21-jscsmk: failed waiting for cluster=\"cluster-candidate-4-21-jscsmk\" in resourcegroup=\"rg-candidate-4-21-fnczl99xh8kl\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c896c3fa-e9be-48ff-abf7-d53fe9674288\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c896c3fa-e9be-48ff-abf7-d53fe9674288\",\n \"name\": \"c896c3fa-e9be-48ff-abf7-d53fe9674288\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:27.440763508Z\",\n \"endTime\": \"2026-08-25T16:40:28.603771658Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e88e20\u003e: \n failed to create HCP cluster ea-listhfgh7b: failed waiting for cluster=\"ea-listhfgh7b\" in resourcegroup=\"ea-list-6p4sr9k5q7bf\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62436e44-f35d-4583-a471-e3b8c1476144\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62436e44-f35d-4583-a471-e3b8c1476144\",\n \"name\": \"62436e44-f35d-4583-a471-e3b8c1476144\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:30.392692104Z\",\n \"endTime\": \"2026-08-25T14:42:33.727154801Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e88e20\u003e: \n failed to create HCP cluster ea-listhfgh7b: failed waiting for cluster=\"ea-listhfgh7b\" in resourcegroup=\"ea-list-6p4sr9k5q7bf\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62436e44-f35d-4583-a471-e3b8c1476144\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62436e44-f35d-4583-a471-e3b8c1476144\",\n \"name\": \"62436e44-f35d-4583-a471-e3b8c1476144\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:30.392692104Z\",\n \"endTime\": \"2026-08-25T14:42:33.727154801Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b8c1c0\u003e: \n failed to create HCP cluster ea-listwl9qrq: failed waiting for cluster=\"ea-listwl9qrq\" in resourcegroup=\"ea-list-cs4nbh2vqlnx\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee1124f8-4c79-44f0-8dc1-bd5636fec70e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee1124f8-4c79-44f0-8dc1-bd5636fec70e\",\n \"name\": \"ee1124f8-4c79-44f0-8dc1-bd5636fec70e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:19.900770335Z\",\n \"endTime\": \"2026-08-25T11:20:27.863710783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b8c1c0\u003e: \n failed to create HCP cluster ea-listwl9qrq: failed waiting for cluster=\"ea-listwl9qrq\" in resourcegroup=\"ea-list-cs4nbh2vqlnx\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee1124f8-4c79-44f0-8dc1-bd5636fec70e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee1124f8-4c79-44f0-8dc1-bd5636fec70e\",\n \"name\": \"ee1124f8-4c79-44f0-8dc1-bd5636fec70e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:19.900770335Z\",\n \"endTime\": \"2026-08-25T11:20:27.863710783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"2501774b6fdfd7c419479e578ae84628be587408415c6fa7b689447828db6eaa","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"de16d3bee3a53f0651e2d553859d03cd36da7aee381249015bf8db424e02b9bb","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"31a5fbbcc1ff5e782a4df57bfa09d31a765c01a8df5db6afadfacac658e620d6","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"c4caf661a44dbe4ab9a819e2a4f4d4a40e4840ddd13380deac01741d243deeb9","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"cc82f7ed850ed911e19f3f8933725fdb7d487b5ae31fdcadcf8ffd1df92eee0f","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"22a2ec90b8e4c03b7eb4b7cf7a664ff8afb21270d440786b9593f19fac9b96a1","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":1.6666666666666667,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,5,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc8b80\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-r67n76nc45fp\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38a6194-1bb0-4fa8-91b4-9d9b52f87625\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38a6194-1bb0-4fa8-91b4-9d9b52f87625\",\n \"name\": \"f38a6194-1bb0-4fa8-91b4-9d9b52f87625\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:31.043336048Z\",\n \"endTime\": \"2026-08-25T19:39:32.419104707Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc8b80\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-r67n76nc45fp\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38a6194-1bb0-4fa8-91b4-9d9b52f87625\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38a6194-1bb0-4fa8-91b4-9d9b52f87625\",\n \"name\": \"f38a6194-1bb0-4fa8-91b4-9d9b52f87625\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:31.043336048Z\",\n \"endTime\": \"2026-08-25T19:39:32.419104707Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-m94cqb294gjz/cluster-candidate-5-0-cd7r5j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd6160\u003e: \n failed to create HCP cluster cluster-candidate-5-0-cd7r5j: failed waiting for cluster=\"cluster-candidate-5-0-cd7r5j\" in resourcegroup=\"rg-candidate-5-0-m94cqb294gjz\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5a5c184-48e5-41b3-82f5-d53463fe130e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5a5c184-48e5-41b3-82f5-d53463fe130e\",\n \"name\": \"f5a5c184-48e5-41b3-82f5-d53463fe130e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:32.285332628Z\",\n \"endTime\": \"2026-08-25T19:41:32.407207308Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-m94cqb294gjz/cluster-candidate-5-0-cd7r5j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd6160\u003e: \n failed to create HCP cluster cluster-candidate-5-0-cd7r5j: failed waiting for cluster=\"cluster-candidate-5-0-cd7r5j\" in resourcegroup=\"rg-candidate-5-0-m94cqb294gjz\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5a5c184-48e5-41b3-82f5-d53463fe130e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5a5c184-48e5-41b3-82f5-d53463fe130e\",\n \"name\": \"f5a5c184-48e5-41b3-82f5-d53463fe130e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:32.285332628Z\",\n \"endTime\": \"2026-08-25T19:41:32.407207308Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c409e0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-g7htcsxqgjh8\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dffc5a1b-8a6e-474f-9d2b-542515a5266b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dffc5a1b-8a6e-474f-9d2b-542515a5266b\",\n \"name\": \"dffc5a1b-8a6e-474f-9d2b-542515a5266b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:51.512570533Z\",\n \"endTime\": \"2026-08-25T11:23:58.061151727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c409e0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-g7htcsxqgjh8\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dffc5a1b-8a6e-474f-9d2b-542515a5266b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dffc5a1b-8a6e-474f-9d2b-542515a5266b\",\n \"name\": \"dffc5a1b-8a6e-474f-9d2b-542515a5266b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:51.512570533Z\",\n \"endTime\": \"2026-08-25T11:23:58.061151727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"6ae7c340a3a6185150f01fedd02fdfde8bcb2b6e79e7c90419ba5cf8f2b0a0c6","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"874e564fd64d9955e95921340933ec68b79b24e1dc50ad4f794673936e060215","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"0fa98435175007c5311cd1a7a6087ef4be15b52a462c15b286367a764bab1677","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"32446f0653d184d03231d5fec01e7294672fb079eed4f9359b944cae9896c2fc","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"c5a29c38a564aa794f98663d402b32a7d56cfc809fa2b5066f123163d7b250c5","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4b645ca11afad5b79ec9dae25eb364818cdce5c3fe97c8a6de8581921ac37e5b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":1.6666666666666667,"anchor_occurrences":5,"after_last_push_count":90,"trend_counts":[0,1,4,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":87,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001064300\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-frc9zfk9zfsm\" to finish creating: GET https://rp.j8006016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e3bd41f8-53bf-48c8-bf55-aef332079de9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e3bd41f8-53bf-48c8-bf55-aef332079de9\",\n \"name\": \"e3bd41f8-53bf-48c8-bf55-aef332079de9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:14:17.602366967Z\",\n \"endTime\": \"2026-08-25T20:33:22.375472132Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001064300\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-frc9zfk9zfsm\" to finish creating: GET https://rp.j8006016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e3bd41f8-53bf-48c8-bf55-aef332079de9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e3bd41f8-53bf-48c8-bf55-aef332079de9\",\n \"name\": \"e3bd41f8-53bf-48c8-bf55-aef332079de9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T20:14:17.602366967Z\",\n \"endTime\": \"2026-08-25T20:33:22.375472132Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff2780\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6hgvwqwzvj5t\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2eba795a-0e4a-4269-8001-ab35fdb2081e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2eba795a-0e4a-4269-8001-ab35fdb2081e\",\n \"name\": \"2eba795a-0e4a-4269-8001-ab35fdb2081e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:39.227173208Z\",\n \"endTime\": \"2026-08-25T16:40:48.587074479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff2780\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6hgvwqwzvj5t\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2eba795a-0e4a-4269-8001-ab35fdb2081e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2eba795a-0e4a-4269-8001-ab35fdb2081e\",\n \"name\": \"2eba795a-0e4a-4269-8001-ab35fdb2081e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:39.227173208Z\",\n \"endTime\": \"2026-08-25T16:40:48.587074479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001002420\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-t2kb5kkgwfmt\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\",\n \"name\": \"eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:26.515094904Z\",\n \"endTime\": \"2026-08-25T16:40:28.648310137Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001002420\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-t2kb5kkgwfmt\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\",\n \"name\": \"eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:26.515094904Z\",\n \"endTime\": \"2026-08-25T16:40:28.648310137Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092332915538006016","occurred_at":"2026-08-25T19:26:55Z","signature_id":"d239a15620cfa3dd6d584d3305274301db680da3872d1a184d5bb2bd6c0477d0","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"460e7645261b38bfd4a8103a388c161c308276756d9b8ea62787b9b5009d2135","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"4e02273c25987feb6c9bf4b5b1d1d5fd920f82e15dc281a89db33f4a9aed76a9","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","occurred_at":"2026-08-25T07:07:02Z","signature_id":"6d98542146857385e0ad2f780a80a61651f234191e5005f754beacbb7ea51a34","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"9806f10f5f28de5b459cf7d30f0fa456f7ab916323bce3ad88103d4bb5b0c102","pr_number":6591,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c210f344f8b31315095a1c725d401f06aa7258ee61130005e0b9120ecf2ff4a8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,5,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c72900\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8h5h4xz7mbfk\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54b206e2-551d-4d64-9bdd-e7dc0862599a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54b206e2-551d-4d64-9bdd-e7dc0862599a\",\n \"name\": \"54b206e2-551d-4d64-9bdd-e7dc0862599a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.60892922Z\",\n \"endTime\": \"2026-08-25T20:07:46.977733317Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c72900\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8h5h4xz7mbfk\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54b206e2-551d-4d64-9bdd-e7dc0862599a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54b206e2-551d-4d64-9bdd-e7dc0862599a\",\n \"name\": \"54b206e2-551d-4d64-9bdd-e7dc0862599a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.60892922Z\",\n \"endTime\": \"2026-08-25T20:07:46.977733317Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013503c0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-k2wkzd68x6rw\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d447528e-038e-4cc3-8a9a-4ae63f05a790\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d447528e-038e-4cc3-8a9a-4ae63f05a790\",\n \"name\": \"d447528e-038e-4cc3-8a9a-4ae63f05a790\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.9471217Z\",\n \"endTime\": \"2026-08-25T20:07:47.037810744Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013503c0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-k2wkzd68x6rw\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d447528e-038e-4cc3-8a9a-4ae63f05a790\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d447528e-038e-4cc3-8a9a-4ae63f05a790\",\n \"name\": \"d447528e-038e-4cc3-8a9a-4ae63f05a790\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.9471217Z\",\n \"endTime\": \"2026-08-25T20:07:47.037810744Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c008e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qmfzw8r72h4d\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1d79c0f-6079-47fa-84e7-db57d11d1583\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1d79c0f-6079-47fa-84e7-db57d11d1583\",\n \"name\": \"b1d79c0f-6079-47fa-84e7-db57d11d1583\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:51:05.668136899Z\",\n \"endTime\": \"2026-08-25T20:10:09.078300161Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c008e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qmfzw8r72h4d\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1d79c0f-6079-47fa-84e7-db57d11d1583\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1d79c0f-6079-47fa-84e7-db57d11d1583\",\n \"name\": \"b1d79c0f-6079-47fa-84e7-db57d11d1583\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:51:05.668136899Z\",\n \"endTime\": \"2026-08-25T20:10:09.078300161Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"05e9d38ab4a3aaad4a0b5371ed4af41ae1885a11dde78c40b966902d6bf39c3e","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"4ba0a65bf49c66d944c03681270514049220b21efd9126ab47abc5555d712dc1","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"62d830a8043675e8a90f508cb3493ce60a38c6b2ad4c7bb07df6c3de3aa1da2c","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"b13e609f7295dd74da4f9e8e1f1a399934f7c35335c10bd5927666f26ed38ac2","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"da7749385b9e21b7d26b42f57364958d539d80f06df4643564a35c6623b5bb10","pr_number":6681,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0fa837367648e2c6015295472e0f8df164e966eed931bdff87d5784d41e8c7f8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":1.6666666666666667,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,0,4,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-18T14:39:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e26160\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-q672lrjjt7xx\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ba6b631-6240-4773-bfa1-ea8d27930011\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ba6b631-6240-4773-bfa1-ea8d27930011\",\n \"name\": \"5ba6b631-6240-4773-bfa1-ea8d27930011\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:59.654390099Z\",\n \"endTime\": \"2026-08-25T19:38:02.42081904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e26160\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-q672lrjjt7xx\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ba6b631-6240-4773-bfa1-ea8d27930011\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ba6b631-6240-4773-bfa1-ea8d27930011\",\n \"name\": \"5ba6b631-6240-4773-bfa1-ea8d27930011\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:59.654390099Z\",\n \"endTime\": \"2026-08-25T19:38:02.42081904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-84dql4hbsnbj/cluster-candidate-5-0-2lchcj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aaf8a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-2lchcj: failed waiting for cluster=\"cluster-candidate-5-0-2lchcj\" in resourcegroup=\"rg-candidate-5-0-84dql4hbsnbj\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f71506a-af2f-4b60-985d-00bd566e1074\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f71506a-af2f-4b60-985d-00bd566e1074\",\n \"name\": \"5f71506a-af2f-4b60-985d-00bd566e1074\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:18.647011306Z\",\n \"endTime\": \"2026-08-25T19:42:23.606424611Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-84dql4hbsnbj/cluster-candidate-5-0-2lchcj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aaf8a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-2lchcj: failed waiting for cluster=\"cluster-candidate-5-0-2lchcj\" in resourcegroup=\"rg-candidate-5-0-84dql4hbsnbj\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f71506a-af2f-4b60-985d-00bd566e1074\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f71506a-af2f-4b60-985d-00bd566e1074\",\n \"name\": \"5f71506a-af2f-4b60-985d-00bd566e1074\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:18.647011306Z\",\n \"endTime\": \"2026-08-25T19:42:23.606424611Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-vs25bgt56m2n/cluster-candidate-4-22-mbk6dn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101e4e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-mbk6dn: failed waiting for cluster=\"cluster-candidate-4-22-mbk6dn\" in resourcegroup=\"rg-candidate-4-22-vs25bgt56m2n\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c94972c0-8210-4d51-960d-183f4fd8e840\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c94972c0-8210-4d51-960d-183f4fd8e840\",\n \"name\": \"c94972c0-8210-4d51-960d-183f4fd8e840\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:41.789123755Z\",\n \"endTime\": \"2026-08-25T19:36:42.565693183Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-vs25bgt56m2n/cluster-candidate-4-22-mbk6dn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101e4e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-mbk6dn: failed waiting for cluster=\"cluster-candidate-4-22-mbk6dn\" in resourcegroup=\"rg-candidate-4-22-vs25bgt56m2n\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c94972c0-8210-4d51-960d-183f4fd8e840\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c94972c0-8210-4d51-960d-183f4fd8e840\",\n \"name\": \"c94972c0-8210-4d51-960d-183f4fd8e840\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:41.789123755Z\",\n \"endTime\": \"2026-08-25T19:36:42.565693183Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"91a343c747731a46478c011c994a808882abeec84c6530eb04b1878042920e4e","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"dbe010bd3eef880599620e7f2c75170adabcb8631a1d8b6c5fd42aa908a7aadb","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314723616821248","occurred_at":"2026-08-25T18:14:37Z","signature_id":"365ceaf0468738d194ab343ea8045834af13aae002f5e53c5e41f490e916f1cc","pr_number":6682,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"25985bba716906e38dc1ab2e46edefe47ee71312f5d30ecca45c85fd1a612dae","pr_number":6630,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":1.6666666666666667,"anchor_occurrences":4,"after_last_push_count":29,"trend_counts":[0,2,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":126,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":4}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T20:35:51Z\n Ended: 2026-08-25T20:50:51Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8006016-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.200:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8006016/providers/microsoft.monitor/accounts/hcps-j8006016\", namespace=\"ocm-arohcpci01-2sdqj69vrnefvnc73ip8kheeoc5sggq6-ea-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hbnhn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sdqj69vrnefvnc73ip8kheeoc5sggq6-ea-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T08:17:26Z\n Ended: 2026-08-25T08:33:27Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-rzqj4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T08:17:26Z\n Ended: 2026-08-25T08:36:25Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-rzqj4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T18:01:25Z\n Ended: 2026-08-24T18:34:22Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T18:01:25Z\n Ended: 2026-08-24T18:34:22Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092332915538006016","occurred_at":"2026-08-25T19:26:55Z","signature_id":"80bee8326b201de86b8f3803385ac59d6fc120eccef01c065c343643ce8ecd8f","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","occurred_at":"2026-08-25T07:07:02Z","signature_id":"1d97f68d37328cfbf8faf6e5077141064b5bf086d525d3989deb1486a0606173","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"97ee0c31f908f1dbb20c9b6757ed42d4855cc07c6e4c69aed451babbcc2b480c","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"99e8b0c7fce35f618653d3b9144ebffd6644e1f77bf1ee7a7e41e85ca3b0c8eb","pr_number":6591,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"da0050d2b241d5b64bca4e3ea0c184bfa4a846f096a3f477d0e09b223d10f5d6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.25,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,0,4,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-22T00:10:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0a1a0\u003e: \n failed to create HCP cluster ea-listmdvtnb: failed waiting for cluster=\"ea-listmdvtnb\" in resourcegroup=\"ea-list-fwskcqw558j7\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09211a76-ef74-4d73-bfcd-1348d7d2680c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09211a76-ef74-4d73-bfcd-1348d7d2680c\",\n \"name\": \"09211a76-ef74-4d73-bfcd-1348d7d2680c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:23.028754386Z\",\n \"endTime\": \"2026-08-25T19:39:32.394595575Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0a1a0\u003e: \n failed to create HCP cluster ea-listmdvtnb: failed waiting for cluster=\"ea-listmdvtnb\" in resourcegroup=\"ea-list-fwskcqw558j7\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09211a76-ef74-4d73-bfcd-1348d7d2680c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09211a76-ef74-4d73-bfcd-1348d7d2680c\",\n \"name\": \"09211a76-ef74-4d73-bfcd-1348d7d2680c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:23.028754386Z\",\n \"endTime\": \"2026-08-25T19:39:32.394595575Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a84540\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qsmsgbz4r9bl\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb200329-6a1f-4203-89e8-8bd77fdd326b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb200329-6a1f-4203-89e8-8bd77fdd326b\",\n \"name\": \"eb200329-6a1f-4203-89e8-8bd77fdd326b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:10.235142906Z\",\n \"endTime\": \"2026-08-25T11:22:17.989968047Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a84540\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qsmsgbz4r9bl\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb200329-6a1f-4203-89e8-8bd77fdd326b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb200329-6a1f-4203-89e8-8bd77fdd326b\",\n \"name\": \"eb200329-6a1f-4203-89e8-8bd77fdd326b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:10.235142906Z\",\n \"endTime\": \"2026-08-25T11:22:17.989968047Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00069e2c0\u003e: \n failed to create HCP cluster np-dg-minor-zd5jtq: failed waiting for cluster=\"np-dg-minor-zd5jtq\" in resourcegroup=\"rg-np-dg-minor-zd5jtq-kkl7vl8r2p4z\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\",\n \"name\": \"20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:30.680263097Z\",\n \"endTime\": \"2026-08-25T11:20:37.924037446Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00069e2c0\u003e: \n failed to create HCP cluster np-dg-minor-zd5jtq: failed waiting for cluster=\"np-dg-minor-zd5jtq\" in resourcegroup=\"rg-np-dg-minor-zd5jtq-kkl7vl8r2p4z\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\",\n \"name\": \"20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:30.680263097Z\",\n \"endTime\": \"2026-08-25T11:20:37.924037446Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"8ad26a920569e563149923a5bcd439d71e32d4914bb330af2790f80b14322048","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"1b1a509582594e9612da11037861a2814fe682ec6e6e3e82e70696765264d057","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"83e815c22c5353bc66d4aad1ca57dc77c126e34a396f19d17d4961aa8e291e1b","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"79463fb4058b95541bcd55a0c46b6475494bc057241f46f47f685884fbe58103","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"34e193d1e971458756c5fedbfac4e10a25751030aa42d3492b58c65f0ceda0fd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.25,"anchor_occurrences":4,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,2,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-10"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-14T20:39:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-pg2742k6drr6/cluster-candidate-4-20-s9697v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00121c780\u003e: \n failed to create HCP cluster cluster-candidate-4-20-s9697v: failed waiting for cluster=\"cluster-candidate-4-20-s9697v\" in resourcegroup=\"rg-candidate-4-20-pg2742k6drr6\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18bd3e8a-96dd-4c6e-9e86-32f76567eee8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18bd3e8a-96dd-4c6e-9e86-32f76567eee8\",\n \"name\": \"18bd3e8a-96dd-4c6e-9e86-32f76567eee8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:26:38.293191515Z\",\n \"endTime\": \"2026-08-25T15:45:40.878934392Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-pg2742k6drr6/cluster-candidate-4-20-s9697v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00121c780\u003e: \n failed to create HCP cluster cluster-candidate-4-20-s9697v: failed waiting for cluster=\"cluster-candidate-4-20-s9697v\" in resourcegroup=\"rg-candidate-4-20-pg2742k6drr6\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18bd3e8a-96dd-4c6e-9e86-32f76567eee8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18bd3e8a-96dd-4c6e-9e86-32f76567eee8\",\n \"name\": \"18bd3e8a-96dd-4c6e-9e86-32f76567eee8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:26:38.293191515Z\",\n \"endTime\": \"2026-08-25T15:45:40.878934392Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706780\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jnnzd8dv8sc2\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4c5630e-4920-46ba-ba8e-4a4b51caf88c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4c5630e-4920-46ba-ba8e-4a4b51caf88c\",\n \"name\": \"d4c5630e-4920-46ba-ba8e-4a4b51caf88c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:53.022759327Z\",\n \"endTime\": \"2026-08-25T11:22:55.91053294Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706780\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jnnzd8dv8sc2\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4c5630e-4920-46ba-ba8e-4a4b51caf88c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4c5630e-4920-46ba-ba8e-4a4b51caf88c\",\n \"name\": \"d4c5630e-4920-46ba-ba8e-4a4b51caf88c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:53.022759327Z\",\n \"endTime\": \"2026-08-25T11:22:55.91053294Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:304]: failed to create HCP cluster np-noedge-7pzvlw\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e28760\u003e: \n failed to create HCP cluster np-noedge-7pzvlw: failed waiting for cluster=\"np-noedge-7pzvlw\" in resourcegroup=\"rg-np-noedge-7pzvlw-hmhh49x6v5k9\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n \"name\": \"d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:29:01.361816726Z\",\n \"endTime\": \"2026-08-24T21:48:06.132587418Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:304]: failed to create HCP cluster np-noedge-7pzvlw\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e28760\u003e: \n failed to create HCP cluster np-noedge-7pzvlw: failed waiting for cluster=\"np-noedge-7pzvlw\" in resourcegroup=\"rg-np-noedge-7pzvlw-hmhh49x6v5k9\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n \"name\": \"d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:29:01.361816726Z\",\n \"endTime\": \"2026-08-24T21:48:06.132587418Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"fe4f032da53c9249c0554d3122ef5a19cf72dc9f02c7bcd2d72076136ebe7044","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"9021161863b438a4babb0dca655669309db8e7ffa0c5f505c949b37f07510077","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"7467fcd7d827463de8cbe0d7b5f551f437c25ba6ae7fc7de2730705826346049","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"ca98190d0c65ef7e152d8e004d3f56a8bcff3a1f40939b19e5d08cfc9a992c4b","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b889d809a654deaf1fcd0d6bef27feaf62c1f89d458297965bd85e6944d16a40","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,4,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-17T17:42:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cd4b20\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7nfq89d68b2j\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b32b4b1-4670-40ea-8086-f9466f12ce3e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b32b4b1-4670-40ea-8086-f9466f12ce3e\",\n \"name\": \"6b32b4b1-4670-40ea-8086-f9466f12ce3e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.362546319Z\",\n \"endTime\": \"2026-08-25T11:24:07.656848362Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, cluster-node-tuning-operator, community-operators-catalog, olm-operator, cluster-storage-operator, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, catalog-operator, redhat-marketplace-catalog, cluster-policy-controller, packageserver, dns-operator, ingress-operator, cluster-image-registry-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cd4b20\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7nfq89d68b2j\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b32b4b1-4670-40ea-8086-f9466f12ce3e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b32b4b1-4670-40ea-8086-f9466f12ce3e\",\n \"name\": \"6b32b4b1-4670-40ea-8086-f9466f12ce3e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.362546319Z\",\n \"endTime\": \"2026-08-25T11:24:07.656848362Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, cluster-node-tuning-operator, community-operators-catalog, olm-operator, cluster-storage-operator, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, catalog-operator, redhat-marketplace-catalog, cluster-policy-controller, packageserver, dns-operator, ingress-operator, cluster-image-registry-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001116340\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bjqmncjbznvt\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e0d347-b695-455c-b828-66deb96d7452\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e0d347-b695-455c-b828-66deb96d7452\",\n \"name\": \"64e0d347-b695-455c-b828-66deb96d7452\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.579389204Z\",\n \"endTime\": \"2026-08-25T11:24:07.68094221Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, olm-operator, openshift-route-controller-manager, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, catalog-operator, certified-operators-catalog, packageserver, cluster-network-operator, ingress-operator, cluster-version-operator, openshift-apiserver, dns-operator, cluster-policy-controller, redhat-marketplace-catalog, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001116340\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bjqmncjbznvt\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e0d347-b695-455c-b828-66deb96d7452\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e0d347-b695-455c-b828-66deb96d7452\",\n \"name\": \"64e0d347-b695-455c-b828-66deb96d7452\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.579389204Z\",\n \"endTime\": \"2026-08-25T11:24:07.68094221Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, olm-operator, openshift-route-controller-manager, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, catalog-operator, certified-operators-catalog, packageserver, cluster-network-operator, ingress-operator, cluster-version-operator, openshift-apiserver, dns-operator, cluster-policy-controller, redhat-marketplace-catalog, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f38340\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-4zrvgxxs9sbf\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\",\n \"name\": \"b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:16.036728583Z\",\n \"endTime\": \"2026-08-25T11:24:17.88745643Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, openshift-route-controller-manager, redhat-operators-catalog, olm-operator, community-operators-catalog, cluster-policy-controller, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-storage-operator, packageserver, hosted-cluster-config-operator, ingress-operator, csi-snapshot-controller-operator, openshift-controller-manager, certified-operators-catalog, cluster-version-operator, cluster-network-operator, dns-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f38340\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-4zrvgxxs9sbf\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\",\n \"name\": \"b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:16.036728583Z\",\n \"endTime\": \"2026-08-25T11:24:17.88745643Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, openshift-route-controller-manager, redhat-operators-catalog, olm-operator, community-operators-catalog, cluster-policy-controller, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-storage-operator, packageserver, hosted-cluster-config-operator, ingress-operator, csi-snapshot-controller-operator, openshift-controller-manager, certified-operators-catalog, cluster-version-operator, cluster-network-operator, dns-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"15cec8044c988cc97a4d3ec832848e5522b08071d86565ff2ec35915e8e8e5ea","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"8e4e26a914a87830fbedc93c1a377a82ad141c9089c89b9edbacc2b4d4d5a21f","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"a6d52fac7766cec3d727e2f0d94855f4d0b82cda86272b0009927fee77388e73","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"0502e7bea814f2dea497a2fefaef20aef7c9a61fc571ac4894bd795f56d24e38","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6fc3be35c0207564b1dc5633bee2c60ed2e3e90031e5c78b08b3a7bcef7800ec","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.25,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,3,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ddcf40\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-q2q7d86xnzgs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\",\n \"name\": \"97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:39.021640766Z\",\n \"endTime\": \"2026-08-25T19:37:42.379207389Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ddcf40\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-q2q7d86xnzgs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\",\n \"name\": \"97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:39.021640766Z\",\n \"endTime\": \"2026-08-25T19:37:42.379207389Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-c4b48j\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001892580\u003e: \n failed to create HCP cluster cluster-zstream-4-21-c4b48j: failed waiting for cluster=\"cluster-zstream-4-21-c4b48j\" in resourcegroup=\"rg-zstream-upgrade-4-21-tjwbnczmk66w\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd2277c-c818-44a4-91cb-db69efb21df4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd2277c-c818-44a4-91cb-db69efb21df4\",\n \"name\": \"5dd2277c-c818-44a4-91cb-db69efb21df4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:24:22.258032156Z\",\n \"endTime\": \"2026-08-25T16:43:28.791772827Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-c4b48j\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001892580\u003e: \n failed to create HCP cluster cluster-zstream-4-21-c4b48j: failed waiting for cluster=\"cluster-zstream-4-21-c4b48j\" in resourcegroup=\"rg-zstream-upgrade-4-21-tjwbnczmk66w\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd2277c-c818-44a4-91cb-db69efb21df4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd2277c-c818-44a4-91cb-db69efb21df4\",\n \"name\": \"5dd2277c-c818-44a4-91cb-db69efb21df4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:24:22.258032156Z\",\n \"endTime\": \"2026-08-25T16:43:28.791772827Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zgp8947jk9nr/cluster-candidate-4-22-8d2tgf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e64200\u003e: \n failed to create HCP cluster cluster-candidate-4-22-8d2tgf: failed waiting for cluster=\"cluster-candidate-4-22-8d2tgf\" in resourcegroup=\"rg-candidate-4-22-zgp8947jk9nr\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fe159a5-0698-4edb-aeeb-bac8baa0530e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fe159a5-0698-4edb-aeeb-bac8baa0530e\",\n \"name\": \"8fe159a5-0698-4edb-aeeb-bac8baa0530e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.272254875Z\",\n \"endTime\": \"2026-08-25T11:21:35.930400552Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zgp8947jk9nr/cluster-candidate-4-22-8d2tgf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e64200\u003e: \n failed to create HCP cluster cluster-candidate-4-22-8d2tgf: failed waiting for cluster=\"cluster-candidate-4-22-8d2tgf\" in resourcegroup=\"rg-candidate-4-22-zgp8947jk9nr\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fe159a5-0698-4edb-aeeb-bac8baa0530e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fe159a5-0698-4edb-aeeb-bac8baa0530e\",\n \"name\": \"8fe159a5-0698-4edb-aeeb-bac8baa0530e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.272254875Z\",\n \"endTime\": \"2026-08-25T11:21:35.930400552Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"84131327fae01288ea8861e4a967044b615e020a15959c0d9a099127d951ae13","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","signature_id":"4cde9318d023ca11a0bb6154c219327b20248fc8365ff58ea50734ae7b25de59","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"541a9bf4087000257fc777c8bd4137d002f9e23cd4994e52825872d346f25bdb","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c3db9c7d221b5f14f4cb052b432def48559d4c9a3619db1db26cc57a99142ed9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.25,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,1,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-21T04:39:49Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0afc0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gvltc2gr7wfl\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a8b4b1e-639e-425c-8c76-d099661899e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a8b4b1e-639e-425c-8c76-d099661899e0\",\n \"name\": \"9a8b4b1e-639e-425c-8c76-d099661899e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:24:12.827625738Z\",\n \"endTime\": \"2026-08-25T14:43:14.39497302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0afc0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gvltc2gr7wfl\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a8b4b1e-639e-425c-8c76-d099661899e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a8b4b1e-639e-425c-8c76-d099661899e0\",\n \"name\": \"9a8b4b1e-639e-425c-8c76-d099661899e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:24:12.827625738Z\",\n \"endTime\": \"2026-08-25T14:43:14.39497302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011e26e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf47q9x8cxz7\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812a5a7d-5729-4953-b916-1cbe8aba438f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812a5a7d-5729-4953-b916-1cbe8aba438f\",\n \"name\": \"812a5a7d-5729-4953-b916-1cbe8aba438f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:21.177808774Z\",\n \"endTime\": \"2026-08-25T14:41:23.66756422Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011e26e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf47q9x8cxz7\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812a5a7d-5729-4953-b916-1cbe8aba438f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812a5a7d-5729-4953-b916-1cbe8aba438f\",\n \"name\": \"812a5a7d-5729-4953-b916-1cbe8aba438f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:21.177808774Z\",\n \"endTime\": \"2026-08-25T14:41:23.66756422Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--wkl8qfr9l4cs/cluster-hs-pre-jtp7jx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009c0960\u003e: \n failed to create HCP cluster cluster-hs-pre-jtp7jx: failed waiting for cluster=\"cluster-hs-pre-jtp7jx\" in resourcegroup=\"rg-hs-pre--wkl8qfr9l4cs\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\",\n \"name\": \"df253c01-fd54-4465-9725-47d0942d268e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:18:48.695053949Z\",\n \"endTime\": \"2026-08-24T15:37:57.993163664Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--wkl8qfr9l4cs/cluster-hs-pre-jtp7jx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009c0960\u003e: \n failed to create HCP cluster cluster-hs-pre-jtp7jx: failed waiting for cluster=\"cluster-hs-pre-jtp7jx\" in resourcegroup=\"rg-hs-pre--wkl8qfr9l4cs\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\",\n \"name\": \"df253c01-fd54-4465-9725-47d0942d268e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:18:48.695053949Z\",\n \"endTime\": \"2026-08-24T15:37:57.993163664Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"b217f98ca261620a2196fa549c1345074f0ad31609adefc4d0a95125d87e751a","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"f0717dee0439ce4b9da20978f82a9a18a9e1f1a19a95e531972424323361a844","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"113a28e0e41ddb16e2d1b2e27f386f97c1c16af7bfd0bd1391cd74394fd64a44","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"30d5e903df58307358d5dbc7b4047e4f537ca0b7b5e66487f298253e6e91fb1f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.25,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,3,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-17T22:25:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c0e640\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-58h6tmctnmgt\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0aad29d-cf26-4c83-b0f3-4c189b3e6419\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0aad29d-cf26-4c83-b0f3-4c189b3e6419\",\n \"name\": \"d0aad29d-cf26-4c83-b0f3-4c189b3e6419\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:02.364791294Z\",\n \"endTime\": \"2026-08-25T19:38:02.432181787Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c0e640\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-58h6tmctnmgt\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0aad29d-cf26-4c83-b0f3-4c189b3e6419\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0aad29d-cf26-4c83-b0f3-4c189b3e6419\",\n \"name\": \"d0aad29d-cf26-4c83-b0f3-4c189b3e6419\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:02.364791294Z\",\n \"endTime\": \"2026-08-25T19:38:02.432181787Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b67620\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-wgbj289pcnlt\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37f858c7-a07c-43e4-a020-589372bc5756\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37f858c7-a07c-43e4-a020-589372bc5756\",\n \"name\": \"37f858c7-a07c-43e4-a020-589372bc5756\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:54:06.405705309Z\",\n \"endTime\": \"2026-08-25T15:13:09.352514019Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b67620\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-wgbj289pcnlt\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37f858c7-a07c-43e4-a020-589372bc5756\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37f858c7-a07c-43e4-a020-589372bc5756\",\n \"name\": \"37f858c7-a07c-43e4-a020-589372bc5756\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:54:06.405705309Z\",\n \"endTime\": \"2026-08-25T15:13:09.352514019Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-6nw9hm with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000892700\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-6nw9hm: failed waiting for cluster=\"np-version-upgrade-cluster-6nw9hm\" in resourcegroup=\"rg-np-version-upgrade-6nw9hm-sbc4vzg69snt\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd8368f0-8487-43cb-9ffb-d72a8e31a349\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd8368f0-8487-43cb-9ffb-d72a8e31a349\",\n \"name\": \"dd8368f0-8487-43cb-9ffb-d72a8e31a349\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:18.056611915Z\",\n \"endTime\": \"2026-08-25T11:32:25.82846965Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-6nw9hm with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000892700\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-6nw9hm: failed waiting for cluster=\"np-version-upgrade-cluster-6nw9hm\" in resourcegroup=\"rg-np-version-upgrade-6nw9hm-sbc4vzg69snt\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd8368f0-8487-43cb-9ffb-d72a8e31a349\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd8368f0-8487-43cb-9ffb-d72a8e31a349\",\n \"name\": \"dd8368f0-8487-43cb-9ffb-d72a8e31a349\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:18.056611915Z\",\n \"endTime\": \"2026-08-25T11:32:25.82846965Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"6873e149feb92978c0e6c51d500f8132748dcf6414954a9cbceb9d163428d9a0","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"9f0220de1cfd60808a1dbdb143dba6ec11917303d1f81188a2549135945ac704","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"d833ff037f5488f1fab9b945d4f55d14924d6973d61766c5187da39c788703a1","pr_number":6618,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.25,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,1,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":22,"prior_last_seen_at":"2026-08-22T14:52:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:153]: failed to scale up node pool np-update-nodes from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0024223c0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-gld8nsfdc76t\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-gld8nsfdc76t, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:153]: failed to scale up node pool np-update-nodes from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0024223c0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-gld8nsfdc76t\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-gld8nsfdc76t, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0033fc180\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jz5t9hfw7w26\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jz5t9hfw7w26, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0033fc180\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jz5t9hfw7w26\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jz5t9hfw7w26, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00396d0e0\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00396d0e0\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092313647266140160","occurred_at":"2026-08-25T18:10:21Z","signature_id":"b81c2e95d57f9e709b06188bf80fa3ab504036a41d889fef881537e3c942592c","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"99c112613fb0611797f784fb621f4a92a4be1186e7f636e286eaba4372997bdb","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"dfb206f2ba6bc7bc3ce13c32e974065647bae3407bafdce87be1e26f94b6936e","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.25,"anchor_occurrences":3,"after_last_push_count":48,"trend_counts":[0,1,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":192,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":3}],"full_error_samples":["alert fired 14 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T17:23:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.142:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn70pm4stdfpd62ob77dhptdq67h82-k7b0n1g7e9t6i9r\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-d2656\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn70pm4stdfpd62ob77dhptdq67h82-k7b0n1g7e9t6i9r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-08-25T17:23:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.142:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn70pm4stdfpd62ob77dhptdq67h82-k7b0n1g7e9t6i9r\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-d2656\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn70pm4stdfpd62ob77dhptdq67h82-k7b0n1g7e9t6i9r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-08-25T17:29:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-08-25T17:29:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-08-25T17:29:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-08-25T17:29:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-08-25T17:29:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-08-25T17:29:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-08-25T17:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-08-25T17:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-08-25T17:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-08-25T17:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-08-25T17:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-08-25T17:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T08:31:29Z\n Ended: 2026-08-25T08:38:27Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-rzqj4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T18:15:21Z\n Ended: 2026-08-24T18:36:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"e028319f9187bbe822ec5100ba5bf10f82963cc9e819d8d10a3fec4e07cba99c","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","occurred_at":"2026-08-25T07:07:02Z","signature_id":"8c886dd26eb0284df24c3f192d77269e9116e884fbce7b0f37949a6fdc5f288d","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"21c20c66c7c9aa8660ddbc1f0e9827cf7f85ba5f2e371e49fb9660d2202ea0a2","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1991ea49f3ca2d6d94a2581eca784f9c57feb75f5c693bb159399a0906185730","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,3,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cfa540\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-6b79jbxxk2jv\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51cea064-8c25-47d5-9e4c-98159ead62b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51cea064-8c25-47d5-9e4c-98159ead62b6\",\n \"name\": \"51cea064-8c25-47d5-9e4c-98159ead62b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:45.544982989Z\",\n \"endTime\": \"2026-08-25T11:23:47.560554329Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, cluster-version-operator, cluster-network-operator, olm-operator, cluster-storage-operator, redhat-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, ingress-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-image-registry-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cfa540\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-6b79jbxxk2jv\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51cea064-8c25-47d5-9e4c-98159ead62b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51cea064-8c25-47d5-9e4c-98159ead62b6\",\n \"name\": \"51cea064-8c25-47d5-9e4c-98159ead62b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:45.544982989Z\",\n \"endTime\": \"2026-08-25T11:23:47.560554329Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, cluster-version-operator, cluster-network-operator, olm-operator, cluster-storage-operator, redhat-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, ingress-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-image-registry-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-wmc2th\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c80de0\u003e: \n failed to create HCP cluster np-noedge-wmc2th: failed waiting for cluster=\"np-noedge-wmc2th\" in resourcegroup=\"rg-np-noedge-wmc2th-55l76kt7fdfw\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/599e72ae-426e-413c-a454-0c456b2c46e6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/599e72ae-426e-413c-a454-0c456b2c46e6\",\n \"name\": \"599e72ae-426e-413c-a454-0c456b2c46e6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:46.955311835Z\",\n \"endTime\": \"2026-08-25T11:23:47.543149183Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, openshift-controller-manager, hosted-cluster-config-operator, ingress-operator, packageserver, cluster-version-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, olm-operator, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, azure-cloud-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, certified-operators-catalog, cluster-network-operator, catalog-operator, redhat-marketplace-catalog, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-wmc2th\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c80de0\u003e: \n failed to create HCP cluster np-noedge-wmc2th: failed waiting for cluster=\"np-noedge-wmc2th\" in resourcegroup=\"rg-np-noedge-wmc2th-55l76kt7fdfw\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/599e72ae-426e-413c-a454-0c456b2c46e6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/599e72ae-426e-413c-a454-0c456b2c46e6\",\n \"name\": \"599e72ae-426e-413c-a454-0c456b2c46e6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:46.955311835Z\",\n \"endTime\": \"2026-08-25T11:23:47.543149183Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, openshift-controller-manager, hosted-cluster-config-operator, ingress-operator, packageserver, cluster-version-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, olm-operator, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, azure-cloud-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, certified-operators-catalog, cluster-network-operator, catalog-operator, redhat-marketplace-catalog, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00134ebc0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-7p6xhpjkj2d7\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9b87905-c782-40c9-95b4-fab4bb1cb211\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9b87905-c782-40c9-95b4-fab4bb1cb211\",\n \"name\": \"e9b87905-c782-40c9-95b4-fab4bb1cb211\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:49.498742711Z\",\n \"endTime\": \"2026-08-25T11:23:52.96330385Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, cluster-policy-controller, dns-operator, catalog-operator, openshift-route-controller-manager, cluster-node-tuning-operator, redhat-operators-catalog, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, packageserver, azure-cloud-controller-manager, openshift-controller-manager, certified-operators-catalog, olm-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00134ebc0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-7p6xhpjkj2d7\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9b87905-c782-40c9-95b4-fab4bb1cb211\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9b87905-c782-40c9-95b4-fab4bb1cb211\",\n \"name\": \"e9b87905-c782-40c9-95b4-fab4bb1cb211\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:49.498742711Z\",\n \"endTime\": \"2026-08-25T11:23:52.96330385Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, cluster-policy-controller, dns-operator, catalog-operator, openshift-route-controller-manager, cluster-node-tuning-operator, redhat-operators-catalog, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, packageserver, azure-cloud-controller-manager, openshift-controller-manager, certified-operators-catalog, olm-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"7962adc60e87afd58df615180673875e6038c2e6c21e761c06a8e2250c897c04","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"baf1c4495aac4315f07ab15d045d6bf7c36c718b7b6f2252cb6b1dd46b38088c","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"4ca6751eeab4072d0b147aaed4435bad054118427342ce0f828034310536b0c2","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13af4eebb33ef90402931e7ab13148cdff6daac3fb3b395a1841590fdaff6d91","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,3,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e94fa0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-w7gfkxf8nw4v\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa0882c-9021-414b-b83f-d1f980679679\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa0882c-9021-414b-b83f-d1f980679679\",\n \"name\": \"8aa0882c-9021-414b-b83f-d1f980679679\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.25563444Z\",\n \"endTime\": \"2026-08-25T19:36:50.662196769Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-network-operator, ingress-operator, packageserver, dns-operator, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e94fa0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-w7gfkxf8nw4v\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa0882c-9021-414b-b83f-d1f980679679\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa0882c-9021-414b-b83f-d1f980679679\",\n \"name\": \"8aa0882c-9021-414b-b83f-d1f980679679\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.25563444Z\",\n \"endTime\": \"2026-08-25T19:36:50.662196769Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-network-operator, ingress-operator, packageserver, dns-operator, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001d5620\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nt76rfflxn6j\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01fbd967-26b2-47f3-ac40-3c302a3e7965\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01fbd967-26b2-47f3-ac40-3c302a3e7965\",\n \"name\": \"01fbd967-26b2-47f3-ac40-3c302a3e7965\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.227475903Z\",\n \"endTime\": \"2026-08-25T19:36:50.710544904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, dns-operator, ingress-operator, packageserver, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001d5620\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nt76rfflxn6j\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01fbd967-26b2-47f3-ac40-3c302a3e7965\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01fbd967-26b2-47f3-ac40-3c302a3e7965\",\n \"name\": \"01fbd967-26b2-47f3-ac40-3c302a3e7965\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.227475903Z\",\n \"endTime\": \"2026-08-25T19:36:50.710544904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, dns-operator, ingress-operator, packageserver, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-qclt2j\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012c2120\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-qclt2j: failed waiting for cluster=\"cp-ystream-upgrade-4-21-qclt2j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-2mbkqtdbwbbj\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c79cbefa-a0f4-4710-a616-f42a2f2efab3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c79cbefa-a0f4-4710-a616-f42a2f2efab3\",\n \"name\": \"c79cbefa-a0f4-4710-a616-f42a2f2efab3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:09.836508695Z\",\n \"endTime\": \"2026-08-25T14:42:13.731155696Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, dns-operator, csi-snapshot-controller-operator, cluster-storage-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-qclt2j\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012c2120\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-qclt2j: failed waiting for cluster=\"cp-ystream-upgrade-4-21-qclt2j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-2mbkqtdbwbbj\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c79cbefa-a0f4-4710-a616-f42a2f2efab3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c79cbefa-a0f4-4710-a616-f42a2f2efab3\",\n \"name\": \"c79cbefa-a0f4-4710-a616-f42a2f2efab3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:09.836508695Z\",\n \"endTime\": \"2026-08-25T14:42:13.731155696Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, dns-operator, csi-snapshot-controller-operator, cluster-storage-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"4e3c6fa2936ba28e83ab9d88f6267d56844fa246b84839315cd5a1f94e2ab484","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"73aa3f2260ddb71b0e4372ee5ba731cfabeb239d1e551a06c5cc7e6c31570148","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"26cd31b13da1cd4b784cc2fc7816f909f73ac2fec28631501c0c3e5e8645ce51","pr_number":6641,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b25a155e232dfc779d7fd5ed5fec6315418c0f2c2aacc2afd101b975cf930003","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,3,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--w5dw7vtzvxkx/cluster-hs-pre-dlbp6v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1e760\u003e: \n failed to create HCP cluster cluster-hs-pre-dlbp6v: failed waiting for cluster=\"cluster-hs-pre-dlbp6v\" in resourcegroup=\"rg-hs-pre--w5dw7vtzvxkx\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7246b37-b59e-4bc0-9719-b3713bc167b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7246b37-b59e-4bc0-9719-b3713bc167b2\",\n \"name\": \"f7246b37-b59e-4bc0-9719-b3713bc167b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:47.341416632Z\",\n \"endTime\": \"2026-08-25T19:36:50.700032947Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--w5dw7vtzvxkx/cluster-hs-pre-dlbp6v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1e760\u003e: \n failed to create HCP cluster cluster-hs-pre-dlbp6v: failed waiting for cluster=\"cluster-hs-pre-dlbp6v\" in resourcegroup=\"rg-hs-pre--w5dw7vtzvxkx\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7246b37-b59e-4bc0-9719-b3713bc167b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7246b37-b59e-4bc0-9719-b3713bc167b2\",\n \"name\": \"f7246b37-b59e-4bc0-9719-b3713bc167b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:47.341416632Z\",\n \"endTime\": \"2026-08-25T19:36:50.700032947Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-c2f59xczgxh8/cluster-candidate-4-21-57drd2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001bba960\u003e: \n failed to create HCP cluster cluster-candidate-4-21-57drd2: failed waiting for cluster=\"cluster-candidate-4-21-57drd2\" in resourcegroup=\"rg-candidate-4-21-c2f59xczgxh8\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8baa3d8-e441-4aa4-ad34-789f804408c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8baa3d8-e441-4aa4-ad34-789f804408c7\",\n \"name\": \"d8baa3d8-e441-4aa4-ad34-789f804408c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:18.423236357Z\",\n \"endTime\": \"2026-08-25T11:21:25.878162329Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-c2f59xczgxh8/cluster-candidate-4-21-57drd2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001bba960\u003e: \n failed to create HCP cluster cluster-candidate-4-21-57drd2: failed waiting for cluster=\"cluster-candidate-4-21-57drd2\" in resourcegroup=\"rg-candidate-4-21-c2f59xczgxh8\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8baa3d8-e441-4aa4-ad34-789f804408c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8baa3d8-e441-4aa4-ad34-789f804408c7\",\n \"name\": \"d8baa3d8-e441-4aa4-ad34-789f804408c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:18.423236357Z\",\n \"endTime\": \"2026-08-25T11:21:25.878162329Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-k9tccs with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c61e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-k9tccs: failed waiting for cluster=\"np-version-upgrade-cluster-k9tccs\" in resourcegroup=\"rg-np-version-upgrade-k9tccs-dxkthfx2xcmq\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06b6d562-cc76-49e7-90e9-e0c893ebdaca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06b6d562-cc76-49e7-90e9-e0c893ebdaca\",\n \"name\": \"06b6d562-cc76-49e7-90e9-e0c893ebdaca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.260205158Z\",\n \"endTime\": \"2026-08-25T11:21:35.932791726Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-k9tccs with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c61e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-k9tccs: failed waiting for cluster=\"np-version-upgrade-cluster-k9tccs\" in resourcegroup=\"rg-np-version-upgrade-k9tccs-dxkthfx2xcmq\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06b6d562-cc76-49e7-90e9-e0c893ebdaca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06b6d562-cc76-49e7-90e9-e0c893ebdaca\",\n \"name\": \"06b6d562-cc76-49e7-90e9-e0c893ebdaca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.260205158Z\",\n \"endTime\": \"2026-08-25T11:21:35.932791726Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"620efc4df137449edf6e36bc5ee396e617e4c93be1f20afe74e9e360cbb49723","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"8535cc4178a95a0eda0b65492ec9ecf1ca1fb3239efd1f0b829eea75cb13b88d","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"a3f04b099b92e17296b69b5c0846c2aba1e01c878468735dffc3852678c7ee27","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0f70434aa7280dae0f02a3c793869c46e1b4bd7e7886836d4307001c25d29ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,3,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-27"],"prior_runs_affected":4,"prior_last_seen_at":"2026-07-29T14:33:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2c400\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-v5fxchbb2prs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561d343f-9a10-4a71-ba6a-31ce4d8eda76\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561d343f-9a10-4a71-ba6a-31ce4d8eda76\",\n \"name\": \"561d343f-9a10-4a71-ba6a-31ce4d8eda76\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:21:13.116412138Z\",\n \"endTime\": \"2026-08-25T19:40:22.435169961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2c400\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-v5fxchbb2prs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561d343f-9a10-4a71-ba6a-31ce4d8eda76\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561d343f-9a10-4a71-ba6a-31ce4d8eda76\",\n \"name\": \"561d343f-9a10-4a71-ba6a-31ce4d8eda76\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:21:13.116412138Z\",\n \"endTime\": \"2026-08-25T19:40:22.435169961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b39a0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-g89x2rmfntgq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ed45382-6139-4927-aeb6-b2580f808168\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ed45382-6139-4927-aeb6-b2580f808168\",\n \"name\": \"8ed45382-6139-4927-aeb6-b2580f808168\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:07:09.765576241Z\",\n \"endTime\": \"2026-08-25T11:26:17.763320122Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b39a0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-g89x2rmfntgq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ed45382-6139-4927-aeb6-b2580f808168\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ed45382-6139-4927-aeb6-b2580f808168\",\n \"name\": \"8ed45382-6139-4927-aeb6-b2580f808168\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:07:09.765576241Z\",\n \"endTime\": \"2026-08-25T11:26:17.763320122Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c189e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-r5zg79zklshc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/460dce50-0e02-4f7d-918e-303c53d250cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/460dce50-0e02-4f7d-918e-303c53d250cd\",\n \"name\": \"460dce50-0e02-4f7d-918e-303c53d250cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.369521114Z\",\n \"endTime\": \"2026-08-25T11:24:07.709199647Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c189e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-r5zg79zklshc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/460dce50-0e02-4f7d-918e-303c53d250cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/460dce50-0e02-4f7d-918e-303c53d250cd\",\n \"name\": \"460dce50-0e02-4f7d-918e-303c53d250cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.369521114Z\",\n \"endTime\": \"2026-08-25T11:24:07.709199647Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"b76793c96776650aa6c2c9acf626989ef5411551675fc33c8a57f8cf74578d78","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"e709edfaf5822475a6c3cdc7e438cd913aefedf35c5c4aa2177cb9bc8689a712","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"f1d2bc77809969c73b592e28941870857aa223052d643ab35baa352ec051a614","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"60a907245af237f1fcb597ff0d35bc01af7bd1e3900650496d648ae7c2abbb63","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [dns-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,3,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-vmkxjb\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001438440\u003e: \n failed to create HCP cluster cluster-zstream-4-22-vmkxjb: failed waiting for cluster=\"cluster-zstream-4-22-vmkxjb\" in resourcegroup=\"rg-zstream-upgrade-4-22-dgzc9xl48h57\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88ef8722-4c84-4313-a177-16c92648110e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88ef8722-4c84-4313-a177-16c92648110e\",\n \"name\": \"88ef8722-4c84-4313-a177-16c92648110e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:49.211451249Z\",\n \"endTime\": \"2026-08-25T11:21:55.960423286Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-vmkxjb\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001438440\u003e: \n failed to create HCP cluster cluster-zstream-4-22-vmkxjb: failed waiting for cluster=\"cluster-zstream-4-22-vmkxjb\" in resourcegroup=\"rg-zstream-upgrade-4-22-dgzc9xl48h57\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88ef8722-4c84-4313-a177-16c92648110e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88ef8722-4c84-4313-a177-16c92648110e\",\n \"name\": \"88ef8722-4c84-4313-a177-16c92648110e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:49.211451249Z\",\n \"endTime\": \"2026-08-25T11:21:55.960423286Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3cae0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-jfmt6ghkl4r6\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/276ea232-8f07-40f6-a841-fed05291e9a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/276ea232-8f07-40f6-a841-fed05291e9a5\",\n \"name\": \"276ea232-8f07-40f6-a841-fed05291e9a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.221378186Z\",\n \"endTime\": \"2026-08-25T11:21:35.933589807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3cae0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-jfmt6ghkl4r6\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/276ea232-8f07-40f6-a841-fed05291e9a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/276ea232-8f07-40f6-a841-fed05291e9a5\",\n \"name\": \"276ea232-8f07-40f6-a841-fed05291e9a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.221378186Z\",\n \"endTime\": \"2026-08-25T11:21:35.933589807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-w6h87f\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001936140\u003e: \n failed to create HCP cluster cluster-zstream-4-20-w6h87f: failed waiting for cluster=\"cluster-zstream-4-20-w6h87f\" in resourcegroup=\"rg-zstream-upgrade-4-20-xmx8p29fpbx4\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a563416d-3979-490a-9710-b74e5cae9107\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a563416d-3979-490a-9710-b74e5cae9107\",\n \"name\": \"a563416d-3979-490a-9710-b74e5cae9107\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.203167213Z\",\n \"endTime\": \"2026-08-25T11:21:35.888478672Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-w6h87f\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001936140\u003e: \n failed to create HCP cluster cluster-zstream-4-20-w6h87f: failed waiting for cluster=\"cluster-zstream-4-20-w6h87f\" in resourcegroup=\"rg-zstream-upgrade-4-20-xmx8p29fpbx4\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a563416d-3979-490a-9710-b74e5cae9107\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a563416d-3979-490a-9710-b74e5cae9107\",\n \"name\": \"a563416d-3979-490a-9710-b74e5cae9107\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.203167213Z\",\n \"endTime\": \"2026-08-25T11:21:35.888478672Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"6e634d35cd843e9cfae5b6b27514de0cb7373b65ef36dac104cfb1c90125fcc9","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"a5d387bcc562b88ab6a23d38e6d2aa3de0eb793a989249e8ad448238a04782c1","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"ae1dbdff3202605f5db0ab0f7e502b028b65320bf6e7bd52d1ff183a02717a24","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T09:11:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-fn4s2m\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dea2e0\u003e: \n DNS for route host agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-fn4s2m\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dea2e0\u003e: \n DNS for route host agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005da4a0\u003e: \n DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005da4a0\u003e: \n DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977495384428544","occurred_at":"2026-08-24T19:54:36Z","signature_id":"4a86fd68539d0c65cc2c794f0a3b117d62969a34ae1b535d87cc0cce6d88b528","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091811921836642304","occurred_at":"2026-08-24T08:56:40Z","signature_id":"67e0d9c3e541426a700c2b1097cab5407e75837246f0b2492c0fa4c512883fb9","pr_number":6618,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"56aad0b4262c7cedf5722ba39358ba8c22af49e4bbcaade93736c9c50ce6eb1a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-14T20:39:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00104e420\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-r77s4bbr26ht\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c31bd00-7879-4578-bf29-c883b2f23367\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c31bd00-7879-4578-bf29-c883b2f23367\",\n \"name\": \"8c31bd00-7879-4578-bf29-c883b2f23367\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:33.22593953Z\",\n \"endTime\": \"2026-08-25T19:38:33.945524152Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00104e420\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-r77s4bbr26ht\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c31bd00-7879-4578-bf29-c883b2f23367\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c31bd00-7879-4578-bf29-c883b2f23367\",\n \"name\": \"8c31bd00-7879-4578-bf29-c883b2f23367\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:33.22593953Z\",\n \"endTime\": \"2026-08-25T19:38:33.945524152Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa6640\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-97gvpsltv9sn\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55ca21a5-264c-4b89-812c-1bd654f8f905\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55ca21a5-264c-4b89-812c-1bd654f8f905\",\n \"name\": \"55ca21a5-264c-4b89-812c-1bd654f8f905\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:24:51.992542762Z\",\n \"endTime\": \"2026-08-25T14:43:53.990674833Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa6640\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-97gvpsltv9sn\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55ca21a5-264c-4b89-812c-1bd654f8f905\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55ca21a5-264c-4b89-812c-1bd654f8f905\",\n \"name\": \"55ca21a5-264c-4b89-812c-1bd654f8f905\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:24:51.992542762Z\",\n \"endTime\": \"2026-08-25T14:43:53.990674833Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"42b392a614867f2ce7fd49a1175ca961a846df1640ce64649fc51da9a67204d7","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"f04cb5f20e7c296508246409d131f2f25950ae8cf510b0c95d54eb7e1af46d29","pr_number":6641,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d188724e2ad8b2ff0009817ae8838a4d0cce4a292b1f33e9f843e12ff24e190c","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--8jnkhgqhx7vj/cluster-hs-pre-gcwzlp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ee120\u003e: \n failed to create HCP cluster cluster-hs-pre-gcwzlp: failed waiting for cluster=\"cluster-hs-pre-gcwzlp\" in resourcegroup=\"rg-hs-pre--8jnkhgqhx7vj\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\",\n \"name\": \"aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:58.14501483Z\",\n \"endTime\": \"2026-08-25T19:38:02.43879336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--8jnkhgqhx7vj/cluster-hs-pre-gcwzlp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ee120\u003e: \n failed to create HCP cluster cluster-hs-pre-gcwzlp: failed waiting for cluster=\"cluster-hs-pre-gcwzlp\" in resourcegroup=\"rg-hs-pre--8jnkhgqhx7vj\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\",\n \"name\": \"aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:58.14501483Z\",\n \"endTime\": \"2026-08-25T19:38:02.43879336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff8340\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-qhbwgqnksltm\" to finish creating: GET https://rp.j9625216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/549edf52-d404-4f5d-b33e-34f0219e1814\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/549edf52-d404-4f5d-b33e-34f0219e1814\",\n \"name\": \"549edf52-d404-4f5d-b33e-34f0219e1814\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:41:53.637919079Z\",\n \"endTime\": \"2026-08-25T16:00:58.269163772Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff8340\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-qhbwgqnksltm\" to finish creating: GET https://rp.j9625216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/549edf52-d404-4f5d-b33e-34f0219e1814\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/549edf52-d404-4f5d-b33e-34f0219e1814\",\n \"name\": \"549edf52-d404-4f5d-b33e-34f0219e1814\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:41:53.637919079Z\",\n \"endTime\": \"2026-08-25T16:00:58.269163772Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"7a9b3449c6776fb39f3e7d85c2865848c6849a9fed0d320c413b71edcfe0b968","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","signature_id":"2b07939ee9bb49277e2d643f0c99927b263772f00f1ecc4f749207d5a2800b2e","pr_number":6648,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fc644a81a7601796e8c7e94724517f0056e9382ba0920b2aaa0529c449d6cba7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002200620\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-2pmrbkkbtqsr\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/27a59a4e-4c36-4607-a1b2-03e8280030f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/27a59a4e-4c36-4607-a1b2-03e8280030f6\",\n \"name\": \"27a59a4e-4c36-4607-a1b2-03e8280030f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:13.29802062Z\",\n \"endTime\": \"2026-08-25T14:42:13.632929999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, dns-operator, cluster-version-operator, cluster-storage-operator, certified-operators-catalog, machine-approver, openshift-controller-manager, catalog-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, redhat-operators-catalog, openshift-apiserver, openshift-route-controller-manager, cluster-network-operator, cluster-node-tuning-operator, olm-operator, community-operators-catalog, cluster-policy-controller, hosted-cluster-config-operator, ingress-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002200620\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-2pmrbkkbtqsr\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/27a59a4e-4c36-4607-a1b2-03e8280030f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/27a59a4e-4c36-4607-a1b2-03e8280030f6\",\n \"name\": \"27a59a4e-4c36-4607-a1b2-03e8280030f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:13.29802062Z\",\n \"endTime\": \"2026-08-25T14:42:13.632929999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, dns-operator, cluster-version-operator, cluster-storage-operator, certified-operators-catalog, machine-approver, openshift-controller-manager, catalog-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, redhat-operators-catalog, openshift-apiserver, openshift-route-controller-manager, cluster-network-operator, cluster-node-tuning-operator, olm-operator, community-operators-catalog, cluster-policy-controller, hosted-cluster-config-operator, ingress-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ca240\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-d4mb4xht7542\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\",\n \"name\": \"d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:29.491781901Z\",\n \"endTime\": \"2026-08-25T14:42:33.748439526Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, cluster-node-tuning-operator, cluster-version-operator, cluster-policy-controller, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, catalog-operator, machine-approver, openshift-route-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, packageserver, openshift-controller-manager, dns-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ca240\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-d4mb4xht7542\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\",\n \"name\": \"d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:29.491781901Z\",\n \"endTime\": \"2026-08-25T14:42:33.748439526Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, cluster-node-tuning-operator, cluster-version-operator, cluster-policy-controller, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, catalog-operator, machine-approver, openshift-route-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, packageserver, openshift-controller-manager, dns-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"51019fbbc87d86ea23801cb65819ccf80ec820166053874e8a6fcf0732993915","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"993ce51c1d7313554e2033361bc0c241e0479977c567c64410226f52de9d8006","pr_number":6641,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f135b58213daed6ea6fbc2013a314b004e48b63448926884614927be5a42fc0d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-network-operator, cluster-node-tuning-operator, dns-operator, hosted-cluster-config-operator, ingress-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8e400\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-mvgcw5tqmrmq\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8748ffe8-2b9e-4821-88fd-9115a64acf5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8748ffe8-2b9e-4821-88fd-9115a64acf5f\",\n \"name\": \"8748ffe8-2b9e-4821-88fd-9115a64acf5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:00.213672088Z\",\n \"endTime\": \"2026-08-25T19:38:02.372107389Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, olm-operator, ingress-operator, hosted-cluster-config-operator, dns-operator, packageserver, cluster-node-tuning-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8e400\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-mvgcw5tqmrmq\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8748ffe8-2b9e-4821-88fd-9115a64acf5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8748ffe8-2b9e-4821-88fd-9115a64acf5f\",\n \"name\": \"8748ffe8-2b9e-4821-88fd-9115a64acf5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:00.213672088Z\",\n \"endTime\": \"2026-08-25T19:38:02.372107389Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, olm-operator, ingress-operator, hosted-cluster-config-operator, dns-operator, packageserver, cluster-node-tuning-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad6ee0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vb2vq2d7pgb7\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e043488-17a2-4487-83fe-b621f2697284\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e043488-17a2-4487-83fe-b621f2697284\",\n \"name\": \"9e043488-17a2-4487-83fe-b621f2697284\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:38.82687874Z\",\n \"endTime\": \"2026-08-25T11:22:45.901459654Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, hosted-cluster-config-operator, packageserver, cluster-node-tuning-operator, dns-operator, cluster-network-operator, ingress-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad6ee0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vb2vq2d7pgb7\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e043488-17a2-4487-83fe-b621f2697284\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e043488-17a2-4487-83fe-b621f2697284\",\n \"name\": \"9e043488-17a2-4487-83fe-b621f2697284\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:38.82687874Z\",\n \"endTime\": \"2026-08-25T11:22:45.901459654Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, hosted-cluster-config-operator, packageserver, cluster-node-tuning-operator, dns-operator, cluster-network-operator, ingress-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"01abb2e6b898b7ccf9ba858e60ea62a7872111a48241f292195140bc087ead89","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"08d1216f17f57d3c84a42ade2f863024d6549f84bec76f38daa65fb0c0f271e4","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d6bc6e1fed29949cb7f899cfde9a499bb81c4498d10956c4f31ee3021f08a1b5","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-cgzvsfdzkmd6/cluster-candidate-4-20-gm7v89 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00146c980\u003e: \n failed to create HCP cluster cluster-candidate-4-20-gm7v89: failed waiting for cluster=\"cluster-candidate-4-20-gm7v89\" in resourcegroup=\"rg-candidate-4-20-cgzvsfdzkmd6\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/048ffc99-876d-41b4-8fb3-df2166c912e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/048ffc99-876d-41b4-8fb3-df2166c912e4\",\n \"name\": \"048ffc99-876d-41b4-8fb3-df2166c912e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.870414032Z\",\n \"endTime\": \"2026-08-25T14:42:23.711491128Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, ingress-operator, cluster-image-registry-operator, dns-operator, hosted-cluster-config-operator, packageserver, catalog-operator, cluster-version-operator, cluster-network-operator, csi-snapshot-controller-operator, cluster-policy-controller, cluster-node-tuning-operator, olm-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-storage-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, machine-approver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-cgzvsfdzkmd6/cluster-candidate-4-20-gm7v89 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00146c980\u003e: \n failed to create HCP cluster cluster-candidate-4-20-gm7v89: failed waiting for cluster=\"cluster-candidate-4-20-gm7v89\" in resourcegroup=\"rg-candidate-4-20-cgzvsfdzkmd6\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/048ffc99-876d-41b4-8fb3-df2166c912e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/048ffc99-876d-41b4-8fb3-df2166c912e4\",\n \"name\": \"048ffc99-876d-41b4-8fb3-df2166c912e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.870414032Z\",\n \"endTime\": \"2026-08-25T14:42:23.711491128Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, ingress-operator, cluster-image-registry-operator, dns-operator, hosted-cluster-config-operator, packageserver, catalog-operator, cluster-version-operator, cluster-network-operator, csi-snapshot-controller-operator, cluster-policy-controller, cluster-node-tuning-operator, olm-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-storage-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, machine-approver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c85320\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8h9ptxfrdc67\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff2cef3-78a1-4805-a77d-d4ff614f91e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff2cef3-78a1-4805-a77d-d4ff614f91e4\",\n \"name\": \"0ff2cef3-78a1-4805-a77d-d4ff614f91e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:00:22.611774831Z\",\n \"endTime\": \"2026-08-25T11:19:27.266054327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, csi-snapshot-controller-operator, community-operators-catalog, certified-operators-catalog, catalog-operator, olm-operator, openshift-apiserver, cluster-policy-controller, dns-operator, openshift-route-controller-manager, ignition-server-proxy, packageserver, redhat-operators-catalog, control-plane-pki-operator, cluster-version-operator, cluster-network-operator, cluster-node-tuning-operator, konnectivity-agent, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c85320\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8h9ptxfrdc67\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff2cef3-78a1-4805-a77d-d4ff614f91e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff2cef3-78a1-4805-a77d-d4ff614f91e4\",\n \"name\": \"0ff2cef3-78a1-4805-a77d-d4ff614f91e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:00:22.611774831Z\",\n \"endTime\": \"2026-08-25T11:19:27.266054327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, csi-snapshot-controller-operator, community-operators-catalog, certified-operators-catalog, catalog-operator, olm-operator, openshift-apiserver, cluster-policy-controller, dns-operator, openshift-route-controller-manager, ignition-server-proxy, packageserver, redhat-operators-catalog, control-plane-pki-operator, cluster-version-operator, cluster-network-operator, cluster-node-tuning-operator, konnectivity-agent, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"327150b988523757d3380ba7ee58ec97de40f9671944b2888f7d5817ed0ac98a","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"9f7d9da9375d115c3d27ec74ca79a5ddf3840aa4ba1cb9141a12954a39487c12","pr_number":6648,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1264ce1f65fb1c4e4b03ef0f0b493117b1b9becb99a5e03d914e5ae33ea84790","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-fl7ppg8w76k6/cluster-candidate-5-0-9h5fbf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013b0300\u003e: \n failed to create HCP cluster cluster-candidate-5-0-9h5fbf: failed waiting for cluster=\"cluster-candidate-5-0-9h5fbf\" in resourcegroup=\"rg-candidate-5-0-fl7ppg8w76k6\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7b99ad-c6a9-48e9-9d4c-e880771fde30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7b99ad-c6a9-48e9-9d4c-e880771fde30\",\n \"name\": \"3e7b99ad-c6a9-48e9-9d4c-e880771fde30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:20:49.919147854Z\",\n \"endTime\": \"2026-08-25T14:39:53.656202045Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, packageserver, cluster-storage-operator, dns-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-fl7ppg8w76k6/cluster-candidate-5-0-9h5fbf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013b0300\u003e: \n failed to create HCP cluster cluster-candidate-5-0-9h5fbf: failed waiting for cluster=\"cluster-candidate-5-0-9h5fbf\" in resourcegroup=\"rg-candidate-5-0-fl7ppg8w76k6\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7b99ad-c6a9-48e9-9d4c-e880771fde30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7b99ad-c6a9-48e9-9d4c-e880771fde30\",\n \"name\": \"3e7b99ad-c6a9-48e9-9d4c-e880771fde30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:20:49.919147854Z\",\n \"endTime\": \"2026-08-25T14:39:53.656202045Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, packageserver, cluster-storage-operator, dns-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0b0e0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-57kmhshxtcx2\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b802d9c9-7b78-4adb-ba82-1320188f9c6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b802d9c9-7b78-4adb-ba82-1320188f9c6d\",\n \"name\": \"b802d9c9-7b78-4adb-ba82-1320188f9c6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:19.928553979Z\",\n \"endTime\": \"2026-08-25T11:21:25.9465048Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, cluster-storage-operator, dns-operator, ingress-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, cluster-image-registry-operator, packageserver, olm-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0b0e0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-57kmhshxtcx2\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b802d9c9-7b78-4adb-ba82-1320188f9c6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b802d9c9-7b78-4adb-ba82-1320188f9c6d\",\n \"name\": \"b802d9c9-7b78-4adb-ba82-1320188f9c6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:19.928553979Z\",\n \"endTime\": \"2026-08-25T11:21:25.9465048Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, cluster-storage-operator, dns-operator, ingress-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, cluster-image-registry-operator, packageserver, olm-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"3bb6f89d65a0e5459009870c86357e81a73e7068ce6cf13f3fa374106ca56368","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"01c47822900529458c20e60551800d6a17821b995d4851679b62a387c6efaa2c","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"52b927fb75e1a0e28c1c3ab5099caef3882ee435ade1e450f06515148c6a9d46","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-l7z8bjhz25gx/cluster-candidate-4-21-mj7vzs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000690a00\u003e: \n failed to create HCP cluster cluster-candidate-4-21-mj7vzs: failed waiting for cluster=\"cluster-candidate-4-21-mj7vzs\" in resourcegroup=\"rg-candidate-4-21-l7z8bjhz25gx\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b46df85-5514-4dc0-92e8-a7d1f99eb864\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b46df85-5514-4dc0-92e8-a7d1f99eb864\",\n \"name\": \"6b46df85-5514-4dc0-92e8-a7d1f99eb864\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.273281186Z\",\n \"endTime\": \"2026-08-25T14:42:23.718787141Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-l7z8bjhz25gx/cluster-candidate-4-21-mj7vzs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000690a00\u003e: \n failed to create HCP cluster cluster-candidate-4-21-mj7vzs: failed waiting for cluster=\"cluster-candidate-4-21-mj7vzs\" in resourcegroup=\"rg-candidate-4-21-l7z8bjhz25gx\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b46df85-5514-4dc0-92e8-a7d1f99eb864\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b46df85-5514-4dc0-92e8-a7d1f99eb864\",\n \"name\": \"6b46df85-5514-4dc0-92e8-a7d1f99eb864\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.273281186Z\",\n \"endTime\": \"2026-08-25T14:42:23.718787141Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14940\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-56n9dv9dj6vs\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/877ebaca-f2ad-4e6f-9b78-d067abc099c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/877ebaca-f2ad-4e6f-9b78-d067abc099c8\",\n \"name\": \"877ebaca-f2ad-4e6f-9b78-d067abc099c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:31.771881865Z\",\n \"endTime\": \"2026-08-25T11:24:32.975906881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, cluster-network-operator, certified-operators-catalog, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14940\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-56n9dv9dj6vs\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/877ebaca-f2ad-4e6f-9b78-d067abc099c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/877ebaca-f2ad-4e6f-9b78-d067abc099c8\",\n \"name\": \"877ebaca-f2ad-4e6f-9b78-d067abc099c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:31.771881865Z\",\n \"endTime\": \"2026-08-25T11:24:32.975906881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, cluster-network-operator, certified-operators-catalog, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"7d1767d3e893f8c308c1422c3d2aa396037e232ca87a62275c5cabd76581cee7","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"319f6c3fb2a06c2f17d4aa28d324ad963b4199781ff328a4f220cea67272ca73","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c975965241af31c57ec44c3fff1102f75c167bb738954ccc985f5a6bb7a81cd2","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator, dns-operator, ignition-server-proxy, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001100500\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-29j67455cd7q\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f324379d-3b99-49b4-a8dd-49ba6e3433d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f324379d-3b99-49b4-a8dd-49ba6e3433d2\",\n \"name\": \"f324379d-3b99-49b4-a8dd-49ba6e3433d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.29234345Z\",\n \"endTime\": \"2026-08-25T11:24:07.6554084Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, dns-operator, cluster-storage-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001100500\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-29j67455cd7q\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f324379d-3b99-49b4-a8dd-49ba6e3433d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f324379d-3b99-49b4-a8dd-49ba6e3433d2\",\n \"name\": \"f324379d-3b99-49b4-a8dd-49ba6e3433d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.29234345Z\",\n \"endTime\": \"2026-08-25T11:24:07.6554084Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, dns-operator, cluster-storage-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8c220\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-c2x56bx2tvvb\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8707f304-4ff5-45ff-8354-0739ca14d80f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8707f304-4ff5-45ff-8354-0739ca14d80f\",\n \"name\": \"8707f304-4ff5-45ff-8354-0739ca14d80f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:09.304723426Z\",\n \"endTime\": \"2026-08-25T11:22:13.012989469Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, dns-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8c220\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-c2x56bx2tvvb\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8707f304-4ff5-45ff-8354-0739ca14d80f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8707f304-4ff5-45ff-8354-0739ca14d80f\",\n \"name\": \"8707f304-4ff5-45ff-8354-0739ca14d80f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:09.304723426Z\",\n \"endTime\": \"2026-08-25T11:22:13.012989469Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, dns-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"2883b2d7f8a4d0400992958856aeb456b0ab19cc4ff95a4341bb724e485a3dc6","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"27c6b10bf1757b4f9a4d1318c60bd484d2401ca40c9413ba0f14effcfc7efe38","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2fc8daa06dcd0810c6da5b7d738c6be1c97e759c090dbe32d9900028e598f859","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-17T07:47:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-t7drrl95cd8z/cluster-candidate-4-20-t2lbd9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a36920\u003e: \n failed to create HCP cluster cluster-candidate-4-20-t2lbd9: failed waiting for cluster=\"cluster-candidate-4-20-t2lbd9\" in resourcegroup=\"rg-candidate-4-20-t7drrl95cd8z\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/579deeb9-f336-4f04-8b2e-b7d680266538\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/579deeb9-f336-4f04-8b2e-b7d680266538\",\n \"name\": \"579deeb9-f336-4f04-8b2e-b7d680266538\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:01.56047746Z\",\n \"endTime\": \"2026-08-25T11:32:05.952606176Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-t7drrl95cd8z/cluster-candidate-4-20-t2lbd9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a36920\u003e: \n failed to create HCP cluster cluster-candidate-4-20-t2lbd9: failed waiting for cluster=\"cluster-candidate-4-20-t2lbd9\" in resourcegroup=\"rg-candidate-4-20-t7drrl95cd8z\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/579deeb9-f336-4f04-8b2e-b7d680266538\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/579deeb9-f336-4f04-8b2e-b7d680266538\",\n \"name\": \"579deeb9-f336-4f04-8b2e-b7d680266538\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:01.56047746Z\",\n \"endTime\": \"2026-08-25T11:32:05.952606176Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-zhhnpcphpl7r/cluster-candidate-4-21-tcc8ch should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2aa60\u003e: \n failed to create HCP cluster cluster-candidate-4-21-tcc8ch: failed waiting for cluster=\"cluster-candidate-4-21-tcc8ch\" in resourcegroup=\"rg-candidate-4-21-zhhnpcphpl7r\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06c23c7d-cba1-433b-881e-637949594df3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06c23c7d-cba1-433b-881e-637949594df3\",\n \"name\": \"06c23c7d-cba1-433b-881e-637949594df3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:29.520227726Z\",\n \"endTime\": \"2026-08-25T11:20:37.842143923Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-zhhnpcphpl7r/cluster-candidate-4-21-tcc8ch should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2aa60\u003e: \n failed to create HCP cluster cluster-candidate-4-21-tcc8ch: failed waiting for cluster=\"cluster-candidate-4-21-tcc8ch\" in resourcegroup=\"rg-candidate-4-21-zhhnpcphpl7r\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06c23c7d-cba1-433b-881e-637949594df3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06c23c7d-cba1-433b-881e-637949594df3\",\n \"name\": \"06c23c7d-cba1-433b-881e-637949594df3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:29.520227726Z\",\n \"endTime\": \"2026-08-25T11:20:37.842143923Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"f2cb49f19339d4683d5c2bd5fd290c1a0f9cba1b7b1446ce110a22ee9a0d5a35","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"c23ab9a03bfcf50206e6f81c6de07ad7f3b1320c629158dce17b675a8a9b9d83","pr_number":6670,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bd8944c9560147308a683885d199860a816c24f450fa26dcc2d571dd6c882239","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--sgwjdjgmhg59/cluster-hs-pre-28tpcf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fc840\u003e: \n failed to create HCP cluster cluster-hs-pre-28tpcf: failed waiting for cluster=\"cluster-hs-pre-28tpcf\" in resourcegroup=\"rg-hs-pre--sgwjdjgmhg59\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd3092ff-ff71-4241-9dfd-bed54a05f55a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd3092ff-ff71-4241-9dfd-bed54a05f55a\",\n \"name\": \"bd3092ff-ff71-4241-9dfd-bed54a05f55a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:23:47.826894918Z\",\n \"endTime\": \"2026-08-25T16:42:48.770504975Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--sgwjdjgmhg59/cluster-hs-pre-28tpcf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fc840\u003e: \n failed to create HCP cluster cluster-hs-pre-28tpcf: failed waiting for cluster=\"cluster-hs-pre-28tpcf\" in resourcegroup=\"rg-hs-pre--sgwjdjgmhg59\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd3092ff-ff71-4241-9dfd-bed54a05f55a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd3092ff-ff71-4241-9dfd-bed54a05f55a\",\n \"name\": \"bd3092ff-ff71-4241-9dfd-bed54a05f55a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:23:47.826894918Z\",\n \"endTime\": \"2026-08-25T16:42:48.770504975Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:93]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6cce0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-pcmvmmpvpflg\" to finish creating: GET https://rp.j9594112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52000f23-e3aa-465f-b2e6-2408f772e24f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52000f23-e3aa-465f-b2e6-2408f772e24f\",\n \"name\": \"52000f23-e3aa-465f-b2e6-2408f772e24f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:59:13.55636864Z\",\n \"endTime\": \"2026-08-25T12:18:21.188692138Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:93]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6cce0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-pcmvmmpvpflg\" to finish creating: GET https://rp.j9594112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52000f23-e3aa-465f-b2e6-2408f772e24f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52000f23-e3aa-465f-b2e6-2408f772e24f\",\n \"name\": \"52000f23-e3aa-465f-b2e6-2408f772e24f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:59:13.55636864Z\",\n \"endTime\": \"2026-08-25T12:18:21.188692138Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","signature_id":"b94b017a26d7c85ad3a642438367f43e24dfb07c60186268412be86721621933","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092208871509594112","occurred_at":"2026-08-25T11:14:00Z","signature_id":"85a73bf3207678bf07196fd0dce16f63b867de9d5fd5b8dcc20aa62ad1433a40","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":215,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":169,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000d6a348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-t87xzxghf54p\" to finish deleting: GET https://rp.j8620928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37cc526b-9ea6-4ae1-aecf-c9876a926ca3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37cc526b-9ea6-4ae1-aecf-c9876a926ca3\",\n \"name\": \"37cc526b-9ea6-4ae1-aecf-c9876a926ca3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:09:21.722027891Z\",\n \"endTime\": \"2026-08-25T19:33:29.709053496Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdpegmnfgfgem1607a6eukk8b4p953h still exists (deletion dispatched at 2026-08-25T19:09:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000d6a348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-t87xzxghf54p\" to finish deleting: GET https://rp.j8620928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37cc526b-9ea6-4ae1-aecf-c9876a926ca3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37cc526b-9ea6-4ae1-aecf-c9876a926ca3\",\n \"name\": \"37cc526b-9ea6-4ae1-aecf-c9876a926ca3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:09:21.722027891Z\",\n \"endTime\": \"2026-08-25T19:33:29.709053496Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdpegmnfgfgem1607a6eukk8b4p953h still exists (deletion dispatched at 2026-08-25T19:09:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000e8af48\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mb2sm98hvbm8\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n \"name\": \"76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:14:43.826676557Z\",\n \"endTime\": \"2026-08-24T15:38:50.051171369Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm25j05l3q2f8a1e83cjmn4lda8g still exists (deletion dispatched at 2026-08-24T15:14:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000e8af48\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mb2sm98hvbm8\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n \"name\": \"76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:14:43.826676557Z\",\n \"endTime\": \"2026-08-24T15:38:50.051171369Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm25j05l3q2f8a1e83cjmn4lda8g still exists (deletion dispatched at 2026-08-24T15:14:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314114058620928","occurred_at":"2026-08-25T18:12:12Z","signature_id":"f9759b9996e08511f5b632db38111c4fa4b734c76990458a0d0586bc1dc45bbb","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"53ff086f26d6ee86f70c417c33372b3bc3883205891aacb5f42319d457fb9cf5","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-p2rrzx at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eda880\u003e: \n failed to create HCP cluster np-skip-minor-p2rrzx: failed waiting for cluster=\"np-skip-minor-p2rrzx\" in resourcegroup=\"rg-np-skip-minor-p2rrzx-kmlcxmpxm26q\" to finish creating: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cbb8807-0089-4be6-ba8a-e5eb93f1f153\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/3cbb8807-0089-4be6-ba8a-e5eb93f1f153' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cbb8807-0089-4be6-ba8a-e5eb93f1f153\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-p2rrzx at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eda880\u003e: \n failed to create HCP cluster np-skip-minor-p2rrzx: failed waiting for cluster=\"np-skip-minor-p2rrzx\" in resourcegroup=\"rg-np-skip-minor-p2rrzx-kmlcxmpxm26q\" to finish creating: GET https://rp.j3938560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cbb8807-0089-4be6-ba8a-e5eb93f1f153\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/3cbb8807-0089-4be6-ba8a-e5eb93f1f153' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cbb8807-0089-4be6-ba8a-e5eb93f1f153\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-lpd7ls\" to 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a5c5a0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-lpd7ls\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-nkpr7nplpjkn\" to finish updating: GET https://rp.j8801025.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-lpd7ls\" to 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a5c5a0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-lpd7ls\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-nkpr7nplpjkn\" to finish updating: GET https://rp.j8801025.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092336210943938560","occurred_at":"2026-08-25T19:39:54Z","signature_id":"6fcc04a2c9029bf0e07a18bcf1c07c92e16d1fbf75f00a5c42d1862ffa2fd969","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191713148801025","occurred_at":"2026-08-25T10:02:43Z","signature_id":"04481477e7547c1ab7c165b4ddca30acd8f9d43ad49b8ba03ce0070a336cbe84","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d3766fe34f03f22876c777e51a7be1734ee728533bfb3f28e111bf0b35fcad1b","failure_pattern":"UpdateHCPCluster20260630 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-72g5866h88vj\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-72g5866h88vj, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":61,"prior_last_seen_at":"2026-08-21T15:05:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00110af90\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-72g5866h88vj\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-72g5866h88vj, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00110af90\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-72g5866h88vj\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-72g5866h88vj, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00132c390\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00132c390\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"44f45bd7c197771dae334f78215fd7f0e9a8f7b495a48ad1be4b5f40460c338c","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"78869f45f7134e749985fc5b23dff71b456538a41d70734a716f1e608670ab6c","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":36,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T08:19:22Z\n Ended: 2026-08-25T08:37:23Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8f2abcc4-a7f6-4094-8ae9-551ab00643de\"\n Description: Pod ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-08-25T08:19:22Z\n Ended: 2026-08-25T08:36:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8f2abcc4-a7f6-4094-8ae9-551ab00643de\"\n Description: Pod ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T18:02:20Z\n Ended: 2026-08-24T18:38:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4cebd923-31c4-4c0a-b568-63bf1a281b9a\"\n Description: Pod ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-08-24T18:03:20Z\n Ended: 2026-08-24T18:38:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4cebd923-31c4-4c0a-b568-63bf1a281b9a\"\n Description: Pod ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","occurred_at":"2026-08-25T07:07:02Z","signature_id":"592b27ee04df02dab6c1e953326523943126040c3e195f04af0ccc2cb082c127","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"9d5e8ec654bc2ebd399a25878bada85329c8658827f68d63b0042fb7498d8047","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","failure_pattern":"alert [svc] FleetControllerRetryHotLoop fired","search_query":"alert [svc] FleetControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T22:05:11Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:28:49Z\n Ended: 2026-08-25T15:35:49Z\n Severity: Sev3\n Labels: alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j0144640-svc\", component=\"fleet\", name=\"clustersserviceregistrationcontroller\", severity=\"warning\"\n Description: Fleet controller workqueue clustersserviceregistrationcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T17:38:07Z\n Ended: 2026-08-24T17:45:05Z\n Severity: Sev3\n Labels: alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j0629888-svc\", component=\"fleet\", name=\"clustersserviceregistrationcontroller\", severity=\"warning\"\n Description: Fleet controller workqueue clustersserviceregistrationcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092258099980144640","occurred_at":"2026-08-25T14:29:37Z","signature_id":"3eebb3220fdafc37e59ae5d05cda26d2e052d0d1681fedb42f8fa69446913217","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"3eebb3220fdafc37e59ae5d05cda26d2e052d0d1681fedb42f8fa69446913217","pr_number":6591,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":69,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":229,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":2}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T11:37:26Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-08-25T11:37:26Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-08-25T11:37:26Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-08-25T11:37:26Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.","alert fired 24 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T16:11:08Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:25:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-08-24T16:13:07Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-08-24T16:13:07Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:26:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-08-24T16:17:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-08-24T16:17:09Z\n Ended: 2026-08-24T16:23:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-08-24T16:17:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-08-24T16:17:09Z\n Ended: 2026-08-24T16:29:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092198034401136640","occurred_at":"2026-08-25T10:30:42Z","signature_id":"f21b751e01203e4e27106dcb1932c067722032d928d9e8e29b55092b5008f5d9","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"367941eb104ee306ff1fe196c961bc8c4a584d69e0f31a0ee0696e981fbdf639","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"470aab7d7a3605e8633a637a70d37da16684c6630946576af6d433f851290640","failure_pattern":"alert [svc] UJClusterProvisionErrors1h5m fired","search_query":"alert [svc] UJClusterProvisionErrors1h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors1h5m does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-27"],"prior_runs_affected":18,"prior_last_seen_at":"2026-07-31T12:10:07Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors1h5m does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T11:31:06Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors1h5m\", cluster=\"ci01-j2121984-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2121984/providers/microsoft.monitor/accounts/services-j2121984\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n Description: More than 72% of cluster create (install) operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours. A regional install failure of this magnitude typically points at a shared dependency (e.g. registry, DNS, or ARM) rather than individual clusters.","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T11:29:19Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors1h5m\", cluster=\"ci01-j5488384-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5488384/providers/microsoft.monitor/accounts/services-j5488384\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n Description: More than 72% of cluster create (install) operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours. A regional install failure of this magnitude typically points at a shared dependency (e.g. registry, DNS, or ARM) rather than individual clusters."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"8726d474691b88bafaa41f02f06ab47cee7673893f1cb6c5f94125da19bae517","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"7ed12c6bca5e7dc43cf2acfe3e266ecf9c33df65853c31ec85ae02d6ffec33ff","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fac505f52469073fa6a64250c64e44813d69018a355acebf1fdde81a505c53f0","failure_pattern":"expected 3 nodes to have original label key1=value1","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-21T15:05:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1\nExpected\n \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"db70381daed7f2c0ae7388805235b1692e0241b6bbf9c9fad5adb0318ac4ed06","failure_pattern":"failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"\u003curl\u003e","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.8333333333333334,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-20T15:48:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.22.77:37397-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.22.77:37397-\u003e172.30.0.10:53: i/o timeout","fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.200.17:33911-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.200.17:33911-\u003e172.30.0.10:53: i/o timeout"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","signature_id":"cf8d75a064f159907166271d9eeaa685334da9829b43b0bb83f85eba1df8e3e4","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092189876085592064","occurred_at":"2026-08-25T09:52:51Z","signature_id":"0fe7f6172a7cc0e6e9770da225a113c760ce09d48125d6f7381c144d3873bcb3","pr_number":6669,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6200cb8cea81a44cbffdcbf30f49e1b2e8791793e7e07d52640fca8ceb6c92c0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce01c0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-vm8pwb95swzd\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\",\n \"name\": \"17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:11.32352815Z\",\n \"endTime\": \"2026-08-25T11:22:15.832343372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, cluster-image-registry-operator, dns-operator, kube-storage-version-migrator, cluster-policy-controller, hosted-cluster-config-operator, cluster-storage-operator, catalog-operator, community-operators-catalog, machine-approver, openshift-route-controller-manager, cluster-network-operator, csi-snapshot-controller-operator, packageserver, certified-operators-catalog, openshift-controller-manager, cluster-version-operator, redhat-operators-catalog, openshift-apiserver, ingress-operator, olm-operator, azure-cloud-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce01c0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-vm8pwb95swzd\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\",\n \"name\": \"17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:11.32352815Z\",\n \"endTime\": \"2026-08-25T11:22:15.832343372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, cluster-image-registry-operator, dns-operator, kube-storage-version-migrator, cluster-policy-controller, hosted-cluster-config-operator, cluster-storage-operator, catalog-operator, community-operators-catalog, machine-approver, openshift-route-controller-manager, cluster-network-operator, csi-snapshot-controller-operator, packageserver, certified-operators-catalog, openshift-controller-manager, cluster-version-operator, redhat-operators-catalog, openshift-apiserver, ingress-operator, olm-operator, azure-cloud-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-bxvq84\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a00480\u003e: \n failed to create HCP cluster cluster-zstream-4-21-bxvq84: failed waiting for cluster=\"cluster-zstream-4-21-bxvq84\" in resourcegroup=\"rg-zstream-upgrade-4-21-f772hgcp6g6p\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c10c2eef-8fa9-4a27-b95a-35876b3db418\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c10c2eef-8fa9-4a27-b95a-35876b3db418\",\n \"name\": \"c10c2eef-8fa9-4a27-b95a-35876b3db418\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.100638601Z\",\n \"endTime\": \"2026-08-25T11:21:36.009069993Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-image-registry-operator, catalog-operator, ingress-operator, csi-snapshot-controller-operator, packageserver, machine-approver, azure-cloud-controller-manager, cluster-network-operator, redhat-marketplace-catalog, cluster-node-tuning-operator, hosted-cluster-config-operator, dns-operator, cluster-policy-controller, redhat-operators-catalog, cluster-version-operator, openshift-controller-manager, olm-operator, community-operators-catalog, openshift-route-controller-manager, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-bxvq84\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a00480\u003e: \n failed to create HCP cluster cluster-zstream-4-21-bxvq84: failed waiting for cluster=\"cluster-zstream-4-21-bxvq84\" in resourcegroup=\"rg-zstream-upgrade-4-21-f772hgcp6g6p\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c10c2eef-8fa9-4a27-b95a-35876b3db418\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c10c2eef-8fa9-4a27-b95a-35876b3db418\",\n \"name\": \"c10c2eef-8fa9-4a27-b95a-35876b3db418\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.100638601Z\",\n \"endTime\": \"2026-08-25T11:21:36.009069993Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-image-registry-operator, catalog-operator, ingress-operator, csi-snapshot-controller-operator, packageserver, machine-approver, azure-cloud-controller-manager, cluster-network-operator, redhat-marketplace-catalog, cluster-node-tuning-operator, hosted-cluster-config-operator, dns-operator, cluster-policy-controller, redhat-operators-catalog, cluster-version-operator, openshift-controller-manager, olm-operator, community-operators-catalog, openshift-route-controller-manager, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"84e02a1a3635ca3e59119a735efac8221692eec41952c2200fb85b25108b853d","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"fb329634cdbcd47b225aa1460092969d8489ffd2ea14b20506198098253816f6","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"efea2dcd245af6f421d5d48a15159d0a97c6fcbf251bcc721e928eae3008a031","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-27"],"prior_runs_affected":2,"prior_last_seen_at":"2026-07-28T16:11:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a3c8e0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4z5qknlhfz6j\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00c871f1-c55a-462f-87ca-c17ffa7cfa45\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00c871f1-c55a-462f-87ca-c17ffa7cfa45\",\n \"name\": \"00c871f1-c55a-462f-87ca-c17ffa7cfa45\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:29.653354515Z\",\n \"endTime\": \"2026-08-25T14:42:33.765546046Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a3c8e0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4z5qknlhfz6j\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00c871f1-c55a-462f-87ca-c17ffa7cfa45\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00c871f1-c55a-462f-87ca-c17ffa7cfa45\",\n \"name\": \"00c871f1-c55a-462f-87ca-c17ffa7cfa45\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:29.653354515Z\",\n \"endTime\": \"2026-08-25T14:42:33.765546046Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011468e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-w7786nk5nznp\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9975191b-6ab2-42a8-85c6-83aed94ceaf0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9975191b-6ab2-42a8-85c6-83aed94ceaf0\",\n \"name\": \"9975191b-6ab2-42a8-85c6-83aed94ceaf0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.963379665Z\",\n \"endTime\": \"2026-08-25T14:42:23.660042904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011468e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-w7786nk5nznp\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9975191b-6ab2-42a8-85c6-83aed94ceaf0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9975191b-6ab2-42a8-85c6-83aed94ceaf0\",\n \"name\": \"9975191b-6ab2-42a8-85c6-83aed94ceaf0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.963379665Z\",\n \"endTime\": \"2026-08-25T14:42:23.660042904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"ebe5c6af0803fc1a7dab9b5770745552c68456e64c75e2851d5fb42f7169307d","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"eed43f59ecea085b1ffeb037fb6bd3536093190e7881447ceae828484fef5878","pr_number":6641,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4a2f8bcf8d29c66434783789d75f6d18e6e94158f0e0f35e53c0353bce508345","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T18:19:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e70e20\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wmkndtkx988t\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\",\n \"name\": \"a9d4baef-549e-4561-b748-439453d113af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:49:08.39871397Z\",\n \"endTime\": \"2026-08-24T16:08:11.236941815Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e70e20\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wmkndtkx988t\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\",\n \"name\": \"a9d4baef-549e-4561-b748-439453d113af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:49:08.39871397Z\",\n \"endTime\": \"2026-08-24T16:08:11.236941815Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000408ec0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn9cdfvxcqll\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\",\n \"name\": \"75726762-be88-402f-a564-60c91125dc8b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:47:28.10624954Z\",\n \"endTime\": \"2026-08-24T16:06:31.149377863Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000408ec0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn9cdfvxcqll\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\",\n \"name\": \"75726762-be88-402f-a564-60c91125dc8b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:47:28.10624954Z\",\n \"endTime\": \"2026-08-24T16:06:31.149377863Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"7dd423d6ebe7dc246942b7ca078647aad56aaec8baea7f2d4aa8292a909f5f19","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"d07feba14b1358e1ba91e00ab3fcc20b89813cd03b2520364e81ecc050b70a83","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b393a3822cea98a3f0355403abde765ef631da69f047c1828358b6194172cb6b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-sjzzj9kjw9kf/cluster-candidate-5-0-rjlgtk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3a660\u003e: \n failed to create HCP cluster cluster-candidate-5-0-rjlgtk: failed waiting for cluster=\"cluster-candidate-5-0-rjlgtk\" in resourcegroup=\"rg-candidate-5-0-sjzzj9kjw9kf\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd830d6-347e-4473-bb10-124104e89a73\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd830d6-347e-4473-bb10-124104e89a73\",\n \"name\": \"8dd830d6-347e-4473-bb10-124104e89a73\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:56.05360308Z\",\n \"endTime\": \"2026-08-25T11:23:57.532994811Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-sjzzj9kjw9kf/cluster-candidate-5-0-rjlgtk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3a660\u003e: \n failed to create HCP cluster cluster-candidate-5-0-rjlgtk: failed waiting for cluster=\"cluster-candidate-5-0-rjlgtk\" in resourcegroup=\"rg-candidate-5-0-sjzzj9kjw9kf\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd830d6-347e-4473-bb10-124104e89a73\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd830d6-347e-4473-bb10-124104e89a73\",\n \"name\": \"8dd830d6-347e-4473-bb10-124104e89a73\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:56.05360308Z\",\n \"endTime\": \"2026-08-25T11:23:57.532994811Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-wndrtf\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00132c740\u003e: \n failed to create HCP cluster cluster-zstream-4-20-wndrtf: failed waiting for cluster=\"cluster-zstream-4-20-wndrtf\" in resourcegroup=\"rg-zstream-upgrade-4-20-btw2nf2rz2z4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\",\n \"name\": \"a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.275646571Z\",\n \"endTime\": \"2026-08-25T11:24:07.685798394Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-wndrtf\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00132c740\u003e: \n failed to create HCP cluster cluster-zstream-4-20-wndrtf: failed waiting for cluster=\"cluster-zstream-4-20-wndrtf\" in resourcegroup=\"rg-zstream-upgrade-4-20-btw2nf2rz2z4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\",\n \"name\": \"a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.275646571Z\",\n \"endTime\": \"2026-08-25T11:24:07.685798394Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"28b9c82589a4dda1452e567d40ab21019c6cee221e37e01472de1dab7d5143df","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"f935555cb57f59b6764c10d1c7fa203076af04b63661aa2b31d669f4c7cdb56f","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2ffeafac71db9cc68f648812079b8ded624a6470e7c1704cbbbe007755392f55","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-prc4m6\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c9c3c0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-prc4m6: failed waiting for cluster=\"cluster-zstream-4-20-prc4m6\" in resourcegroup=\"rg-zstream-upgrade-4-20-74gcx95r2n24\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\",\n \"name\": \"7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:20:56.871797957Z\",\n \"endTime\": \"2026-08-25T16:39:58.557618251Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-prc4m6\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c9c3c0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-prc4m6: failed waiting for cluster=\"cluster-zstream-4-20-prc4m6\" in resourcegroup=\"rg-zstream-upgrade-4-20-74gcx95r2n24\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\",\n \"name\": \"7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:20:56.871797957Z\",\n \"endTime\": \"2026-08-25T16:39:58.557618251Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-zgjbj5 with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b1e160\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-zgjbj5: failed waiting for cluster=\"np-version-upgrade-cluster-zgjbj5\" in resourcegroup=\"rg-np-version-upgrade-zgjbj5-5lcrgtsl9x9p\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\",\n \"name\": \"ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:20:59.293835394Z\",\n \"endTime\": \"2026-08-25T16:40:08.618237862Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-zgjbj5 with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b1e160\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-zgjbj5: failed waiting for cluster=\"np-version-upgrade-cluster-zgjbj5\" in resourcegroup=\"rg-np-version-upgrade-zgjbj5-5lcrgtsl9x9p\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\",\n \"name\": \"ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:20:59.293835394Z\",\n \"endTime\": \"2026-08-25T16:40:08.618237862Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"1e71ee8a9468b4add845994e78143a38f98dc804aebda495c2ae55adfc2902b9","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"338f3c10fa302ec66d489a7e252c6f877b54dac7f42009f33bf533ea79c5069d","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T08:55:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-tp6lxr8bfgv2\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/863038dc-ed8c-4013-be96-35e5dd9a5f59\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/863038dc-ed8c-4013-be96-35e5dd9a5f59\",\n \"name\": \"863038dc-ed8c-4013-be96-35e5dd9a5f59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:10:58.76533919Z\",\n \"endTime\": \"2026-08-25T11:29:59.215293237Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-tp6lxr8bfgv2\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/863038dc-ed8c-4013-be96-35e5dd9a5f59\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/863038dc-ed8c-4013-be96-35e5dd9a5f59\",\n \"name\": \"863038dc-ed8c-4013-be96-35e5dd9a5f59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:10:58.76533919Z\",\n \"endTime\": \"2026-08-25T11:29:59.215293237Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"f43c2f2c050759d71b643af6fc429ef7cf204b9a1f2b922e8824120be8851f15","pr_number":6648,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b55233e1f233919ecc579e6967f76823775cbaf35c5ee37e0a8883b46726c324","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AllocationFailed; detail message The VM allocation failed due to an internal error. Please retry later or try deploying to a different location.; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-16T12:18:51Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-24T19:10:07.135Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T19:10:08.078Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T19:10:08.961Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6400512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05\ntime=2026-08-24T19:19:10.348Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05/operationStatuses/08584140082768889498\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/cluster-3vizzf2ahtjlc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6400512-mgmt-2/agentPools/userswft1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AllocationFailed\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft1-38184404-vmss'. Please try alternative sizes 'Standard_E16ds_v6' (zones 2, 3, 1), 'Standard_E16ds_v6' (eastus, zones 1, 2, 3), 'Standard_E16ds_v6' (eastus2, zones 2, 1, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AllocationFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"3\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The VM allocation failed due to an internal error. Please retry later or try deploying to a different location.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091957267166400512","occurred_at":"2026-08-24T18:34:13Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cb3f1af3ae5c6870302b8493e6567a2cb5b04bdc76461d8531d6cdaf4f74aada","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ConflictError; detail message A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-03"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-07T00:57:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e56140\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-update-nodes-hcp-cluster-chtjsv\" in resourcegroup=\"nodepool-update-nodes-sz99wmd6668j\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/nodepool-update-nodes-sz99wmd6668j/providers/Microsoft.Resources/deployments/customer-infra-np-update-nodes-hcp-cluster-chtjsv/operationStatuses/08584139228279467438\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ConflictError\\\",\\r\\n \\\"message\\\": \\\"A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: https://go.microsoft.com/fwlink/?linkid=2147741\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e56140\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-update-nodes-hcp-cluster-chtjsv\" in resourcegroup=\"nodepool-update-nodes-sz99wmd6668j\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/nodepool-update-nodes-sz99wmd6668j/providers/Microsoft.Resources/deployments/customer-infra-np-update-nodes-hcp-cluster-chtjsv/operationStatuses/08584139228279467438\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ConflictError\\\",\\r\\n \\\"message\\\": \\\"A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: https://go.microsoft.com/fwlink/?linkid=2147741\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314114058620928","occurred_at":"2026-08-25T18:12:12Z","signature_id":"6fabb5bf9236add92797cdcc42cf1ddf33539333a8c7650c981d9fd3ebea687d","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bb424b290c173083428a34ac21b9472b0a9be4d1c592b771df2c71e46b269465","failure_pattern":"ERROR CODE: DeploymentFailed; detail code StorageAccountOperationInProgress; detail message An operation is currently performing on this storage account that requires exclusive access.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-24T16:09:49.176Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 description=\"Step infra\\n Kind: ARM\\n Template: templates/mgmt-infra.bicep\\n Parameters: configurations/mgmt-infra.tmpl.bicepparam\"\ntime=2026-08-24T16:09:49.840Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2\ntime=2026-08-24T16:09:50.553Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 deployment=c783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1968512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290\ntime=2026-08-24T16:11:20.899Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/c783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290/operationStatuses/08584140190952500816\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcp-backups-storage\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcpBackupsStorageAccount\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcpBackupsStorageAccount\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"StorageAccountOperationInProgress\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"An operation is currently performing on this storage account that requires exclusive access.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091914307531968512","occurred_at":"2026-08-24T15:43:31Z","signature_id":"d517e51fc5b09c819903bacd247416bcbbe167939922787f3a2438a125b3fdc2","pr_number":6659,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Timeout; detail message Operation is being Canceled due to timeout.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-08-24T07:13:06.981Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-08-24T07:13:09.018Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-08-24T07:13:10.536Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1860736%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59\ntime=2026-08-24T07:29:44.382Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1860736/providers/Microsoft.Resources/deployments/9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59/operationStatuses/08584140512958847265\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1860736/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1860736/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1860736\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091780518591860736","occurred_at":"2026-08-24T06:51:53Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6278,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36e7932077fe0de3697c04fb0ba302a7298b56f122f6c49db035b3d764e4980b","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentStackDeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nodepools (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nodepools (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-24T08:15:25.803Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nodepools stamp=2 description=\"Step nodepools\\n Kind: ARMStack\\n Template: templates/aks-nodepools.bicep\\n Parameters: configurations/mgmt-cluster-nodepools.tmpl.bicepparam\"\ntime=2026-08-24T08:18:16.150Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nodepools stamp=2 err=\"stamp 2: failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\\",\\n \\\"name\\\": \\\"67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\\",\\n \\\"status\\\": \\\"failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: '676c3fe4-b0a1-44f5-8602-c3c05012b379'.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2/providers/Microsoft.Resources/deployments/l4mv3scto7z6edigthkjdr62jbdrgoxoif33nvakxlwvg-260824084ipd0\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"OverconstrainedZonalAllocationRequest\\\",\\n \\\"message\\\": \\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-41975975-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-41975975-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"\\n }\\n ]\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091793135989428224","occurred_at":"2026-08-24T07:42:01Z","signature_id":"497ab79e6fe621da7f982b21f5dd28a45dc2f63297d3ed53abd84200879145f5","pr_number":6447,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"72de96d5fccd2e929d86c234bb6122fb3e96395118e1ddf6c2c21fc7fb224f37","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101a160\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-qmzd7vdsthg7\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1cf37f1-e335-4a31-8459-8927106e9275\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1cf37f1-e335-4a31-8459-8927106e9275\",\n \"name\": \"d1cf37f1-e335-4a31-8459-8927106e9275\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:20.481967932Z\",\n \"endTime\": \"2026-08-25T19:39:22.436659526Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101a160\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-qmzd7vdsthg7\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1cf37f1-e335-4a31-8459-8927106e9275\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1cf37f1-e335-4a31-8459-8927106e9275\",\n \"name\": \"d1cf37f1-e335-4a31-8459-8927106e9275\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:20.481967932Z\",\n \"endTime\": \"2026-08-25T19:39:22.436659526Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"d496820b37cab7fa1ea7b0c47e837d533327b2947405aa862fe6cf0becda824e","pr_number":6683,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2bdf0d0e26486c8baecfb566f0a7ae932a4a2cee72db742fb786cc445dbea21e","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, catalog-operator, certified-operators-catalog, cluster-autoscaler, cluster-image-registry-operator, cluster-network-operator, cluster-node-tuning-operator, cluster-policy-controller, cluster-storage-operator, cluster-version-operator, community-operators-catalog, control-plane-pki-operator, csi-snapshot-controller-operator, dns-operator, hosted-cluster-config-operator, ignition-server, ignition-server-proxy, ingress-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, machine-approver, olm-collect-profiles, olm-operator, openshift-apiserver, openshift-controller-manager, openshift-route-controller-manager, packageserver, redhat-marketplace-catalog, redhat-operators-catalog; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fa820\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-gp4rdpwm7zhd\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ee129a2-c81c-4d02-bea3-92db5bdcc541\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ee129a2-c81c-4d02-bea3-92db5bdcc541\",\n \"name\": \"5ee129a2-c81c-4d02-bea3-92db5bdcc541\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:55.463575249Z\",\n \"endTime\": \"2026-08-25T11:23:57.522671815Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, olm-collect-profiles, cluster-policy-controller, cluster-node-tuning-operator, certified-operators-catalog, cluster-storage-operator, dns-operator, cluster-image-registry-operator, packageserver, hosted-cluster-config-operator, cluster-network-operator, ignition-server-proxy, catalog-operator, openshift-apiserver, ignition-server, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, machine-approver, kube-controller-manager, openshift-controller-manager, csi-snapshot-controller-operator, control-plane-pki-operator, kube-scheduler, cluster-version-operator, cluster-autoscaler, openshift-route-controller-manager, azure-cloud-controller-manager, ingress-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fa820\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-gp4rdpwm7zhd\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ee129a2-c81c-4d02-bea3-92db5bdcc541\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ee129a2-c81c-4d02-bea3-92db5bdcc541\",\n \"name\": \"5ee129a2-c81c-4d02-bea3-92db5bdcc541\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:55.463575249Z\",\n \"endTime\": \"2026-08-25T11:23:57.522671815Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, olm-collect-profiles, cluster-policy-controller, cluster-node-tuning-operator, certified-operators-catalog, cluster-storage-operator, dns-operator, cluster-image-registry-operator, packageserver, hosted-cluster-config-operator, cluster-network-operator, ignition-server-proxy, catalog-operator, openshift-apiserver, ignition-server, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, machine-approver, kube-controller-manager, openshift-controller-manager, csi-snapshot-controller-operator, control-plane-pki-operator, kube-scheduler, cluster-version-operator, cluster-autoscaler, openshift-route-controller-manager, azure-cloud-controller-manager, ingress-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"66ee2372a7d35bafc2a7139acace8a079bd17442b7e81b824c6f13e5ea4be15f","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b3d328e9a97f3f976227e1f892def10b7f6c53bda848b798f9760113f189d2a4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator, ingress-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a2ee20\u003e: \n failed to create HCP cluster np-dg-minor-8wczf9: failed waiting for cluster=\"np-dg-minor-8wczf9\" in resourcegroup=\"rg-np-dg-minor-8wczf9-z98rv7l55r2s\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35e222d0-bdde-4e12-a2e6-3658ebb699b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35e222d0-bdde-4e12-a2e6-3658ebb699b6\",\n \"name\": \"35e222d0-bdde-4e12-a2e6-3658ebb699b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:58.632569496Z\",\n \"endTime\": \"2026-08-25T11:24:07.68601483Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, cluster-image-registry-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a2ee20\u003e: \n failed to create HCP cluster np-dg-minor-8wczf9: failed waiting for cluster=\"np-dg-minor-8wczf9\" in resourcegroup=\"rg-np-dg-minor-8wczf9-z98rv7l55r2s\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35e222d0-bdde-4e12-a2e6-3658ebb699b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35e222d0-bdde-4e12-a2e6-3658ebb699b6\",\n \"name\": \"35e222d0-bdde-4e12-a2e6-3658ebb699b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:58.632569496Z\",\n \"endTime\": \"2026-08-25T11:24:07.68601483Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, cluster-image-registry-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"3286e3629eaf6809d169cb339959b7df55417659272206300cd3912a2d082c76","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"92ae44f2e86af5544dec422ded7c331d2f560f22fa4954538768a565c71a9dbd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e84820\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-b9lzcpbpn5wp\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2bbcb592-de1c-4ea4-8806-5849cdf0735a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2bbcb592-de1c-4ea4-8806-5849cdf0735a\",\n \"name\": \"2bbcb592-de1c-4ea4-8806-5849cdf0735a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.392903067Z\",\n \"endTime\": \"2026-08-25T11:24:07.668839785Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e84820\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-b9lzcpbpn5wp\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2bbcb592-de1c-4ea4-8806-5849cdf0735a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2bbcb592-de1c-4ea4-8806-5849cdf0735a\",\n \"name\": \"2bbcb592-de1c-4ea4-8806-5849cdf0735a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.392903067Z\",\n \"endTime\": \"2026-08-25T11:24:07.668839785Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"a34611ae2bd1afe363e9b85e5d346d6e037222fe5510e7c418c4c0d0a2de08f2","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e339823035a3b42061d546c2e6cbdec3cd57b33bd40fddf5adef300f46092d74","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, openshift-controller-manager; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001081c80\u003e: \n GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7baee5b-db11-4464-aa8e-6e2d27501249\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7baee5b-db11-4464-aa8e-6e2d27501249\",\n \"name\": \"f7baee5b-db11-4464-aa8e-6e2d27501249\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:19.502868721Z\",\n \"endTime\": \"2026-08-25T11:25:27.633996529Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-node-tuning-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001081c80\u003e: \n GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7baee5b-db11-4464-aa8e-6e2d27501249\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7baee5b-db11-4464-aa8e-6e2d27501249\",\n \"name\": \"f7baee5b-db11-4464-aa8e-6e2d27501249\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:19.502868721Z\",\n \"endTime\": \"2026-08-25T11:25:27.633996529Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-node-tuning-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"62ef36a640529db2357933eec4cfc1865780247b38f5766b5fa5e83338b90158","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7be6b09c6bbb28adba99c6de7dfc1604dfce000bc6664dedbae8e8b9b0253fa6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006eefa0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-blffsk8cqz4z\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb49989-b0d0-4959-b0b9-b9f58628cdd6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb49989-b0d0-4959-b0b9-b9f58628cdd6\",\n \"name\": \"7cb49989-b0d0-4959-b0b9-b9f58628cdd6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.01277565Z\",\n \"endTime\": \"2026-08-25T11:24:07.575415707Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006eefa0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-blffsk8cqz4z\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb49989-b0d0-4959-b0b9-b9f58628cdd6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb49989-b0d0-4959-b0b9-b9f58628cdd6\",\n \"name\": \"7cb49989-b0d0-4959-b0b9-b9f58628cdd6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.01277565Z\",\n \"endTime\": \"2026-08-25T11:24:07.575415707Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"20fc32ee45196a9ed01e8c936d2dde739ba00031d099ddd39be043bfecacc1ed","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ab3cc8ce178fa5fcf5b2293012368a7b35c8796690239f588635f09f67db92fe","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007fc3a0\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-7qr482mpm7kt\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7872695-995d-44a9-aff2-649d7b0c3f98\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7872695-995d-44a9-aff2-649d7b0c3f98\",\n \"name\": \"b7872695-995d-44a9-aff2-649d7b0c3f98\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.045350362Z\",\n \"endTime\": \"2026-08-25T11:23:42.92215819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007fc3a0\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-7qr482mpm7kt\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7872695-995d-44a9-aff2-649d7b0c3f98\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7872695-995d-44a9-aff2-649d7b0c3f98\",\n \"name\": \"b7872695-995d-44a9-aff2-649d7b0c3f98\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.045350362Z\",\n \"endTime\": \"2026-08-25T11:23:42.92215819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"f6f2233f1410b658d6612e4e90347e1d367b288e857c9f69f6fd3585012f8aab","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e367f2a3465740bc9b9da68a2f81e3714490aba74464e39ae8e5c3a1ae5bab94","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, kube-storage-version-migrator, machine-approver, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-p7lpzhts95vd/cluster-candidate-4-22-hlfnj9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005ad040\u003e: \n failed to create HCP cluster cluster-candidate-4-22-hlfnj9: failed waiting for cluster=\"cluster-candidate-4-22-hlfnj9\" in resourcegroup=\"rg-candidate-4-22-p7lpzhts95vd\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18015e61-c049-41a2-809f-ad31666a4af5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18015e61-c049-41a2-809f-ad31666a4af5\",\n \"name\": \"18015e61-c049-41a2-809f-ad31666a4af5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:40.338382698Z\",\n \"endTime\": \"2026-08-25T11:20:48.26963606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-image-registry-operator, ingress-operator, kube-storage-version-migrator, catalog-operator, packageserver, certified-operators-catalog, csi-snapshot-controller-operator, community-operators-catalog, cluster-version-operator, azure-cloud-controller-manager, cluster-storage-operator, redhat-operators-catalog, cluster-policy-controller, cluster-network-operator, machine-approver, dns-operator, olm-operator, openshift-apiserver, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-p7lpzhts95vd/cluster-candidate-4-22-hlfnj9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005ad040\u003e: \n failed to create HCP cluster cluster-candidate-4-22-hlfnj9: failed waiting for cluster=\"cluster-candidate-4-22-hlfnj9\" in resourcegroup=\"rg-candidate-4-22-p7lpzhts95vd\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18015e61-c049-41a2-809f-ad31666a4af5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18015e61-c049-41a2-809f-ad31666a4af5\",\n \"name\": \"18015e61-c049-41a2-809f-ad31666a4af5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:40.338382698Z\",\n \"endTime\": \"2026-08-25T11:20:48.26963606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-image-registry-operator, ingress-operator, kube-storage-version-migrator, catalog-operator, packageserver, certified-operators-catalog, csi-snapshot-controller-operator, community-operators-catalog, cluster-version-operator, azure-cloud-controller-manager, cluster-storage-operator, redhat-operators-catalog, cluster-policy-controller, cluster-network-operator, machine-approver, dns-operator, olm-operator, openshift-apiserver, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"312f9db8ee161a9fc497742dfc7136dbb8ed546241173b0109c0ff6c461ac5e3","pr_number":6670,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6e365d99bf1fe7b4ecb0645e26216599f71568b4b9def38c7d2aa9626ca4060b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, kube-storage-version-migrator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6qqtw5xzzhgg/cluster-candidate-5-0-skdztd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae4220\u003e: \n failed to create HCP cluster cluster-candidate-5-0-skdztd: failed waiting for cluster=\"cluster-candidate-5-0-skdztd\" in resourcegroup=\"rg-candidate-5-0-6qqtw5xzzhgg\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\",\n \"name\": \"bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:58.368910284Z\",\n \"endTime\": \"2026-08-25T11:22:03.073725302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, cluster-storage-operator, cluster-version-operator, cluster-policy-controller, csi-snapshot-controller-operator, kube-storage-version-migrator, olm-operator, redhat-marketplace-catalog, cluster-node-tuning-operator, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, community-operators-catalog, ingress-operator, catalog-operator, openshift-route-controller-manager, dns-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6qqtw5xzzhgg/cluster-candidate-5-0-skdztd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae4220\u003e: \n failed to create HCP cluster cluster-candidate-5-0-skdztd: failed waiting for cluster=\"cluster-candidate-5-0-skdztd\" in resourcegroup=\"rg-candidate-5-0-6qqtw5xzzhgg\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\",\n \"name\": \"bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:58.368910284Z\",\n \"endTime\": \"2026-08-25T11:22:03.073725302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, cluster-storage-operator, cluster-version-operator, cluster-policy-controller, csi-snapshot-controller-operator, kube-storage-version-migrator, olm-operator, redhat-marketplace-catalog, cluster-node-tuning-operator, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, community-operators-catalog, ingress-operator, catalog-operator, openshift-route-controller-manager, dns-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"822b64d629b6a8eaf1077aec1bbfe1cde30b863a89d30905f961017f5f5f8da1","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"04ab38fb4caffee001686eb32195645f728f8555d294ffa19de0a9f505223827","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--mdtqq6tfggzt/cluster-hs-pre-5md8sn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138c380\u003e: \n failed to create HCP cluster cluster-hs-pre-5md8sn: failed waiting for cluster=\"cluster-hs-pre-5md8sn\" in resourcegroup=\"rg-hs-pre--mdtqq6tfggzt\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/faa7138f-d176-4225-9189-5771c1682daa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/faa7138f-d176-4225-9189-5771c1682daa\",\n \"name\": \"faa7138f-d176-4225-9189-5771c1682daa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.650546675Z\",\n \"endTime\": \"2026-08-25T11:20:57.345950087Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, ingress-operator, redhat-marketplace-catalog, cluster-policy-controller, azure-cloud-controller-manager, cluster-image-registry-operator, certified-operators-catalog, packageserver, cluster-version-operator, cluster-storage-operator, redhat-operators-catalog, csi-snapshot-controller-operator, openshift-controller-manager, cluster-node-tuning-operator, openshift-apiserver, catalog-operator, community-operators-catalog, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-network-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--mdtqq6tfggzt/cluster-hs-pre-5md8sn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138c380\u003e: \n failed to create HCP cluster cluster-hs-pre-5md8sn: failed waiting for cluster=\"cluster-hs-pre-5md8sn\" in resourcegroup=\"rg-hs-pre--mdtqq6tfggzt\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/faa7138f-d176-4225-9189-5771c1682daa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/faa7138f-d176-4225-9189-5771c1682daa\",\n \"name\": \"faa7138f-d176-4225-9189-5771c1682daa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.650546675Z\",\n \"endTime\": \"2026-08-25T11:20:57.345950087Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, ingress-operator, redhat-marketplace-catalog, cluster-policy-controller, azure-cloud-controller-manager, cluster-image-registry-operator, certified-operators-catalog, packageserver, cluster-version-operator, cluster-storage-operator, redhat-operators-catalog, csi-snapshot-controller-operator, openshift-controller-manager, cluster-node-tuning-operator, openshift-apiserver, catalog-operator, community-operators-catalog, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-network-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"6cfd6ef557fef8c9d185efe4042060278e1497ede40310252790075002d9aac9","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de9ee73e1f42212ffe8bacf6081359c2c391b2c47c68ba07fa4cd11f00e7b015","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-vk6cfm\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001216240\u003e: \n failed to create HCP cluster np-noedge-vk6cfm: failed waiting for cluster=\"np-noedge-vk6cfm\" in resourcegroup=\"rg-np-noedge-vk6cfm-r5z9ggrs4bzc\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d73055c-caa7-4581-b662-b31a5154ad44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d73055c-caa7-4581-b662-b31a5154ad44\",\n \"name\": \"6d73055c-caa7-4581-b662-b31a5154ad44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:40.855489688Z\",\n \"endTime\": \"2026-08-25T11:21:45.894005118Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, packageserver, olm-collect-profiles, community-operators-catalog, azure-cloud-controller-manager, cluster-storage-operator, konnectivity-agent, cluster-autoscaler, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, openshift-apiserver, cluster-image-registry-operator, certified-operators-catalog, capi-provider, openshift-route-controller-manager, cluster-policy-controller, hosted-cluster-config-operator, dns-operator, ignition-server, olm-operator, machine-approver, kube-controller-manager, kube-scheduler, cluster-network-operator, control-plane-pki-operator, openshift-controller-manager, redhat-operators-catalog, cluster-node-tuning-operator, ignition-server-proxy, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-vk6cfm\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001216240\u003e: \n failed to create HCP cluster np-noedge-vk6cfm: failed waiting for cluster=\"np-noedge-vk6cfm\" in resourcegroup=\"rg-np-noedge-vk6cfm-r5z9ggrs4bzc\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d73055c-caa7-4581-b662-b31a5154ad44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d73055c-caa7-4581-b662-b31a5154ad44\",\n \"name\": \"6d73055c-caa7-4581-b662-b31a5154ad44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:40.855489688Z\",\n \"endTime\": \"2026-08-25T11:21:45.894005118Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, packageserver, olm-collect-profiles, community-operators-catalog, azure-cloud-controller-manager, cluster-storage-operator, konnectivity-agent, cluster-autoscaler, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, openshift-apiserver, cluster-image-registry-operator, certified-operators-catalog, capi-provider, openshift-route-controller-manager, cluster-policy-controller, hosted-cluster-config-operator, dns-operator, ignition-server, olm-operator, machine-approver, kube-controller-manager, kube-scheduler, cluster-network-operator, control-plane-pki-operator, openshift-controller-manager, redhat-operators-catalog, cluster-node-tuning-operator, ignition-server-proxy, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"e075d605863206fdc2ccadedfa2c3d789e15cebb06d5f64e9eeed7ecaf6fde31","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"db4713748d0354976aeef8212e3a9f4a93e3ecf500342fbbcb1fbd6a21579f7d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-network-operator, csi-snapshot-controller-operator, dns-operator, hosted-cluster-config-operator, ingress-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea1740\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jm4vsb7vn4sd\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afbb1cd8-1c39-42be-9ff7-787ffc89e069\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afbb1cd8-1c39-42be-9ff7-787ffc89e069\",\n \"name\": \"afbb1cd8-1c39-42be-9ff7-787ffc89e069\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:22.824480944Z\",\n \"endTime\": \"2026-08-25T11:22:27.407570836Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, csi-snapshot-controller-operator, cluster-version-operator, olm-operator, dns-operator, catalog-operator, cluster-storage-operator, ingress-operator, hosted-cluster-config-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea1740\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jm4vsb7vn4sd\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afbb1cd8-1c39-42be-9ff7-787ffc89e069\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afbb1cd8-1c39-42be-9ff7-787ffc89e069\",\n \"name\": \"afbb1cd8-1c39-42be-9ff7-787ffc89e069\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:22.824480944Z\",\n \"endTime\": \"2026-08-25T11:22:27.407570836Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, csi-snapshot-controller-operator, cluster-version-operator, olm-operator, dns-operator, catalog-operator, cluster-storage-operator, ingress-operator, hosted-cluster-config-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"a0b46f2049c857f9b50193bfa943a46b002ac4931192ad3d996324be47f034d8","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a7bb05c760ce6e02186100a06d53cbce3c5ec829fc4f0c3e2e6bbbb1e0d7f73e","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-version-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001095660\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-b4qlj57lcxr8\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\",\n \"name\": \"c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:28.224401746Z\",\n \"endTime\": \"2026-08-25T11:24:37.557730725Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, cluster-network-operator, community-operators-catalog, certified-operators-catalog, cluster-policy-controller, olm-operator, packageserver, redhat-operators-catalog, dns-operator, ingress-operator, cluster-version-operator, openshift-controller-manager, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001095660\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-b4qlj57lcxr8\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\",\n \"name\": \"c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:28.224401746Z\",\n \"endTime\": \"2026-08-25T11:24:37.557730725Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, cluster-network-operator, community-operators-catalog, certified-operators-catalog, cluster-policy-controller, olm-operator, packageserver, redhat-operators-catalog, dns-operator, ingress-operator, cluster-version-operator, openshift-controller-manager, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"e3135c78719260f8b136dccd2d27e839bcb88513840553199c7a5a72c9c78af4","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c97c7e10e9984956816f2978cee06c0347355fb43e2e18dbc518e10a50fd697c","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--2zj5qrnqfwdn/cluster-hs-pre-4n4p2k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007e89c0\u003e: \n failed to create HCP cluster cluster-hs-pre-4n4p2k: failed waiting for cluster=\"cluster-hs-pre-4n4p2k\" in resourcegroup=\"rg-hs-pre--2zj5qrnqfwdn\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7939f0c1-73a2-48bf-80a1-c5a81f59060f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7939f0c1-73a2-48bf-80a1-c5a81f59060f\",\n \"name\": \"7939f0c1-73a2-48bf-80a1-c5a81f59060f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:28.656859519Z\",\n \"endTime\": \"2026-08-25T11:20:37.93508057Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-policy-controller, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, cluster-image-registry-operator, packageserver, certified-operators-catalog, hosted-cluster-config-operator, olm-operator, openshift-route-controller-manager, openshift-controller-manager, catalog-operator, cluster-network-operator, machine-approver, csi-snapshot-controller-operator, community-operators-catalog, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--2zj5qrnqfwdn/cluster-hs-pre-4n4p2k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007e89c0\u003e: \n failed to create HCP cluster cluster-hs-pre-4n4p2k: failed waiting for cluster=\"cluster-hs-pre-4n4p2k\" in resourcegroup=\"rg-hs-pre--2zj5qrnqfwdn\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7939f0c1-73a2-48bf-80a1-c5a81f59060f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7939f0c1-73a2-48bf-80a1-c5a81f59060f\",\n \"name\": \"7939f0c1-73a2-48bf-80a1-c5a81f59060f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:28.656859519Z\",\n \"endTime\": \"2026-08-25T11:20:37.93508057Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-policy-controller, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, cluster-image-registry-operator, packageserver, certified-operators-catalog, hosted-cluster-config-operator, olm-operator, openshift-route-controller-manager, openshift-controller-manager, catalog-operator, cluster-network-operator, machine-approver, csi-snapshot-controller-operator, community-operators-catalog, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"62b66f3cda1a1b03c5a9f22a0c0fb4bfb068df7258a21d08a9456f0d47c984ee","pr_number":6670,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e258b478ac9644b09eb20a3a01941c7945db5504ffd2b81616ca149fbb4626ea","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019fc320\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-pvw94n8jrfrs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17ba9133-3d45-4498-9ddf-cf4f2ab615f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17ba9133-3d45-4498-9ddf-cf4f2ab615f6\",\n \"name\": \"17ba9133-3d45-4498-9ddf-cf4f2ab615f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:50.270808871Z\",\n \"endTime\": \"2026-08-25T19:37:52.396630467Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, csi-snapshot-controller-operator, packageserver, machine-approver, cluster-policy-controller, cluster-image-registry-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-network-operator, ingress-operator, catalog-operator, olm-operator, community-operators-catalog, dns-operator, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019fc320\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-pvw94n8jrfrs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17ba9133-3d45-4498-9ddf-cf4f2ab615f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17ba9133-3d45-4498-9ddf-cf4f2ab615f6\",\n \"name\": \"17ba9133-3d45-4498-9ddf-cf4f2ab615f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:50.270808871Z\",\n \"endTime\": \"2026-08-25T19:37:52.396630467Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, csi-snapshot-controller-operator, packageserver, machine-approver, cluster-policy-controller, cluster-image-registry-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-network-operator, ingress-operator, catalog-operator, olm-operator, community-operators-catalog, dns-operator, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"3a165bfaf5767e237e81c7162c0c48f850249eb7416a5043163724820b2a6d91","pr_number":6683,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"56ac1ffd8f7c84d6800647347b139a137e691a55ff2e3d1faa305a6b07101364","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-x7w9rq with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000218ae0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-x7w9rq: failed waiting for cluster=\"np-version-upgrade-cluster-x7w9rq\" in resourcegroup=\"rg-np-version-upgrade-x7w9rq-dvf8r82dxzzq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\",\n \"name\": \"96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:57.577943839Z\",\n \"endTime\": \"2026-08-25T11:23:57.610020208Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: certified-operators-catalog deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-x7w9rq with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000218ae0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-x7w9rq: failed waiting for cluster=\"np-version-upgrade-cluster-x7w9rq\" in resourcegroup=\"rg-np-version-upgrade-x7w9rq-dvf8r82dxzzq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\",\n \"name\": \"96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:57.577943839Z\",\n \"endTime\": \"2026-08-25T11:23:57.610020208Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: certified-operators-catalog deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"93f75a595fe834ff90d354323b1795c64c6aa1fffff5420185d0cc08b6045965","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ef607a572d4c53f3a2663eb448d653e968a80b7e9c4d5dd17a7cbefc30c64148","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, cluster-node-tuning-operator, cluster-policy-controller, ingress-operator, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008526a0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-x8svbm7w8gvv\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ed00b3-c155-4497-b144-33a7a3897320\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ed00b3-c155-4497-b144-33a7a3897320\",\n \"name\": \"37ed00b3-c155-4497-b144-33a7a3897320\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.292639561Z\",\n \"endTime\": \"2026-08-25T11:21:35.949263958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator, cluster-policy-controller, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008526a0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-x8svbm7w8gvv\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ed00b3-c155-4497-b144-33a7a3897320\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ed00b3-c155-4497-b144-33a7a3897320\",\n \"name\": \"37ed00b3-c155-4497-b144-33a7a3897320\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.292639561Z\",\n \"endTime\": \"2026-08-25T11:21:35.949263958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator, cluster-policy-controller, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"5c82c50cdd5f477dc9bec1b74ec5428943202baab54c500d7530303ea24f62fe","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"11980404af14f04ae347cf052e220be11d0d2393fbd6aa0e5b7916303d89b95b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-k6tv8vdvt6xx/cluster-candidate-4-22-zlxjzv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018feb00\u003e: \n failed to create HCP cluster cluster-candidate-4-22-zlxjzv: failed waiting for cluster=\"cluster-candidate-4-22-zlxjzv\" in resourcegroup=\"rg-candidate-4-22-k6tv8vdvt6xx\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9dd2d4-422b-44d3-9712-ae568d04d7dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9dd2d4-422b-44d3-9712-ae568d04d7dd\",\n \"name\": \"7c9dd2d4-422b-44d3-9712-ae568d04d7dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:38.96838764Z\",\n \"endTime\": \"2026-08-25T11:23:43.158826266Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, ingress-operator, dns-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-k6tv8vdvt6xx/cluster-candidate-4-22-zlxjzv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018feb00\u003e: \n failed to create HCP cluster cluster-candidate-4-22-zlxjzv: failed waiting for cluster=\"cluster-candidate-4-22-zlxjzv\" in resourcegroup=\"rg-candidate-4-22-k6tv8vdvt6xx\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9dd2d4-422b-44d3-9712-ae568d04d7dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9dd2d4-422b-44d3-9712-ae568d04d7dd\",\n \"name\": \"7c9dd2d4-422b-44d3-9712-ae568d04d7dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:38.96838764Z\",\n \"endTime\": \"2026-08-25T11:23:43.158826266Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, ingress-operator, dns-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"2bdb39b170bc1309151a9f820a6a2455edec3be17d92a609195106d19a9d06dc","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"172dca1d402dce2af7dedfe8fcbd440e4917e443080839ac33b06bea5d61d387","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008d98e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-lnqr4972dtpb\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f003e8b0-f989-4c38-8deb-37f35273e17d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f003e8b0-f989-4c38-8deb-37f35273e17d\",\n \"name\": \"f003e8b0-f989-4c38-8deb-37f35273e17d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:20.791598979Z\",\n \"endTime\": \"2026-08-25T19:37:20.738606915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, cluster-network-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008d98e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-lnqr4972dtpb\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f003e8b0-f989-4c38-8deb-37f35273e17d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f003e8b0-f989-4c38-8deb-37f35273e17d\",\n \"name\": \"f003e8b0-f989-4c38-8deb-37f35273e17d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:20.791598979Z\",\n \"endTime\": \"2026-08-25T19:37:20.738606915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, cluster-network-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"5648256268d040d66790c83ce020dd81edbe55daec10ac344f210eb0a2221d9a","pr_number":6554,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"74426a739e4f86dd5411cdf4cc53f0d913b028f0d225517b981efa72c5de466f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, dns-operator, ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b325c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hcdb8hh7hcl9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a36ee1da-d072-4335-bca2-f17838bce647\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a36ee1da-d072-4335-bca2-f17838bce647\",\n \"name\": \"a36ee1da-d072-4335-bca2-f17838bce647\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:49.267992354Z\",\n \"endTime\": \"2026-08-25T11:23:53.017585492Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator, ingress-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b325c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hcdb8hh7hcl9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a36ee1da-d072-4335-bca2-f17838bce647\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a36ee1da-d072-4335-bca2-f17838bce647\",\n \"name\": \"a36ee1da-d072-4335-bca2-f17838bce647\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:49.267992354Z\",\n \"endTime\": \"2026-08-25T11:23:53.017585492Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator, ingress-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"a52aed8bdf10bcdf14463a386f5de9b79532d4654760f91890403f2b46a1a5f7","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e9bb62f46de99c0acdbb4abf9f88a23afd79f058d2b6721fedb3e87d7c10a712","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-jmx49s\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019b2760\u003e: \n failed to create HCP cluster cluster-zstream-5-0-jmx49s: failed waiting for cluster=\"cluster-zstream-5-0-jmx49s\" in resourcegroup=\"rg-zstream-upgrade-5-0-j2n79nx59vfm\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\",\n \"name\": \"f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:27.608916079Z\",\n \"endTime\": \"2026-08-25T11:21:32.925137069Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-jmx49s\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019b2760\u003e: \n failed to create HCP cluster cluster-zstream-5-0-jmx49s: failed waiting for cluster=\"cluster-zstream-5-0-jmx49s\" in resourcegroup=\"rg-zstream-upgrade-5-0-j2n79nx59vfm\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\",\n \"name\": \"f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:27.608916079Z\",\n \"endTime\": \"2026-08-25T11:21:32.925137069Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"d33c4e76ff9a43fcb2716c265a586de405e0886f83620fe7fc6fe9c586dabbef","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7d7f14514a88680ff6615f00bb81c0bec2c6dadd1b601567325ec72490caf5ca","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0a6c0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-9mf2mfq4r2d9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8a52f10-9e35-4b81-8bed-515ddc39f2b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8a52f10-9e35-4b81-8bed-515ddc39f2b4\",\n \"name\": \"d8a52f10-9e35-4b81-8bed-515ddc39f2b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:12.867884764Z\",\n \"endTime\": \"2026-08-25T11:22:17.332591987Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0a6c0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-9mf2mfq4r2d9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8a52f10-9e35-4b81-8bed-515ddc39f2b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8a52f10-9e35-4b81-8bed-515ddc39f2b4\",\n \"name\": \"d8a52f10-9e35-4b81-8bed-515ddc39f2b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:12.867884764Z\",\n \"endTime\": \"2026-08-25T11:22:17.332591987Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"4bd8257d88ff066db3ad8c08e1600658f460390b8e6e515cc65d3100aa511a22","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"247cc84147e4574f2635a831e66f7a46a1eb594df690c18944f1a846c19ad8ca","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, ingress-operator, redhat-marketplace-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018da440\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qd5fftk5xqnx\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c61f442-7e7a-4287-a096-d40849d2499b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c61f442-7e7a-4287-a096-d40849d2499b\",\n \"name\": \"3c61f442-7e7a-4287-a096-d40849d2499b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:37.077702612Z\",\n \"endTime\": \"2026-08-25T16:40:38.597952289Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, ingress-operator, dns-operator, cluster-image-registry-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018da440\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qd5fftk5xqnx\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c61f442-7e7a-4287-a096-d40849d2499b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c61f442-7e7a-4287-a096-d40849d2499b\",\n \"name\": \"3c61f442-7e7a-4287-a096-d40849d2499b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:37.077702612Z\",\n \"endTime\": \"2026-08-25T16:40:38.597952289Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, ingress-operator, dns-operator, cluster-image-registry-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"390df46832f7606456e52cdf4a307b68f19cb093f5c95774b04156c02106bca8","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"279b8dc059ffca13ba2e7632bee0bb38b6b6e79fa3da4d820708453f05c04b29","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, dns-operator, ignition-server-proxy, kube-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00059c3e0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-24c5bjzcljlb\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b21aa181-e22f-4f73-9442-34e06ecf2205\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b21aa181-e22f-4f73-9442-34e06ecf2205\",\n \"name\": \"b21aa181-e22f-4f73-9442-34e06ecf2205\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:55.381643689Z\",\n \"endTime\": \"2026-08-25T11:23:57.677492232Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00059c3e0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-24c5bjzcljlb\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b21aa181-e22f-4f73-9442-34e06ecf2205\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b21aa181-e22f-4f73-9442-34e06ecf2205\",\n \"name\": \"b21aa181-e22f-4f73-9442-34e06ecf2205\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:55.381643689Z\",\n \"endTime\": \"2026-08-25T11:23:57.677492232Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"41cf799b841c41ca021361bf55e60796cb2a72779166c6626d1c1aedefd1fe20","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"65963d58371f4feaf1ead13b45a131ed3c4d34cd31f41683e65892b6e7918355","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, dns-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4xrqktzlmtt5/cluster-candidate-4-22-rc2wqs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019065e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-rc2wqs: failed waiting for cluster=\"cluster-candidate-4-22-rc2wqs\" in resourcegroup=\"rg-candidate-4-22-4xrqktzlmtt5\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af9b261c-a44f-4aca-93ac-ca9eb2668f2a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af9b261c-a44f-4aca-93ac-ca9eb2668f2a\",\n \"name\": \"af9b261c-a44f-4aca-93ac-ca9eb2668f2a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:41.204224191Z\",\n \"endTime\": \"2026-08-25T19:37:43.979682955Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4xrqktzlmtt5/cluster-candidate-4-22-rc2wqs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019065e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-rc2wqs: failed waiting for cluster=\"cluster-candidate-4-22-rc2wqs\" in resourcegroup=\"rg-candidate-4-22-4xrqktzlmtt5\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af9b261c-a44f-4aca-93ac-ca9eb2668f2a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af9b261c-a44f-4aca-93ac-ca9eb2668f2a\",\n \"name\": \"af9b261c-a44f-4aca-93ac-ca9eb2668f2a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:41.204224191Z\",\n \"endTime\": \"2026-08-25T19:37:43.979682955Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"62dce0e08ee026ebe516bd675e439a69e85ef9b00198fc35594656b8245c2bc5","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"951de8af83bafe4594b1180a34b1f825907eff4cdbe9d6ea0e8637301c989320","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001372260\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-b748z7nmqrc2\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4e67768d-4fd8-4d6f-b652-cd3c57cfd546\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4e67768d-4fd8-4d6f-b652-cd3c57cfd546\",\n \"name\": \"4e67768d-4fd8-4d6f-b652-cd3c57cfd546\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:48.323976798Z\",\n \"endTime\": \"2026-08-25T11:21:53.061758244Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001372260\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-b748z7nmqrc2\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4e67768d-4fd8-4d6f-b652-cd3c57cfd546\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4e67768d-4fd8-4d6f-b652-cd3c57cfd546\",\n \"name\": \"4e67768d-4fd8-4d6f-b652-cd3c57cfd546\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:48.323976798Z\",\n \"endTime\": \"2026-08-25T11:21:53.061758244Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"dc93c011273628c3be4689097fa54bae4867b676ee6f408953d5289c92e59a88","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9ac40957e0d202cc6756c5cbbe3d0ef3ac7730a3425dfc91631cac2e91ff702d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-xk9vbv\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001000ca0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-xk9vbv: failed waiting for cluster=\"cluster-zstream-5-0-xk9vbv\" in resourcegroup=\"rg-zstream-upgrade-5-0-9zdkrc5gfnqq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9f8e41-e479-4f6e-9ade-6570233d1d04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9f8e41-e479-4f6e-9ade-6570233d1d04\",\n \"name\": \"7c9f8e41-e479-4f6e-9ade-6570233d1d04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.893167006Z\",\n \"endTime\": \"2026-08-25T11:24:07.716194982Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-xk9vbv\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001000ca0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-xk9vbv: failed waiting for cluster=\"cluster-zstream-5-0-xk9vbv\" in resourcegroup=\"rg-zstream-upgrade-5-0-9zdkrc5gfnqq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9f8e41-e479-4f6e-9ade-6570233d1d04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9f8e41-e479-4f6e-9ade-6570233d1d04\",\n \"name\": \"7c9f8e41-e479-4f6e-9ade-6570233d1d04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.893167006Z\",\n \"endTime\": \"2026-08-25T11:24:07.716194982Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"e70ccddbda0336cf522314d54bd2bb07c61aafcb132de78c1ca60fa8b7fd1d35","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"66285e3570a1d3a91819718028ead75ff34527595051ad0083e72a103f62194a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014868a0\u003e: \n failed to create HCP cluster np-dg-minor-6lfgc2: failed waiting for cluster=\"np-dg-minor-6lfgc2\" in resourcegroup=\"rg-np-dg-minor-6lfgc2-lwtz9m4rnlh9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/169a9726-cf09-492c-bf9a-53259660198c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/169a9726-cf09-492c-bf9a-53259660198c\",\n \"name\": \"169a9726-cf09-492c-bf9a-53259660198c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:53.246951809Z\",\n \"endTime\": \"2026-08-25T11:20:57.403791086Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014868a0\u003e: \n failed to create HCP cluster np-dg-minor-6lfgc2: failed waiting for cluster=\"np-dg-minor-6lfgc2\" in resourcegroup=\"rg-np-dg-minor-6lfgc2-lwtz9m4rnlh9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/169a9726-cf09-492c-bf9a-53259660198c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/169a9726-cf09-492c-bf9a-53259660198c\",\n \"name\": \"169a9726-cf09-492c-bf9a-53259660198c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:53.246951809Z\",\n \"endTime\": \"2026-08-25T11:20:57.403791086Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"09dd934fdb303c274c99f6ad4ed42588a7c9f786ef23d8ba1c2d65fa831cb2e2","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a12a1aaebf551a02f7b6b7033999753bb4e7d2b0f9d00ac7c03c67219d973f16","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00049fd20\u003e: \n failed to create HCP cluster np-downgrade-gwfjf8: failed waiting for cluster=\"np-downgrade-gwfjf8\" in resourcegroup=\"rg-np-downgrade-gwfjf8-7kkgmzwhgxk9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a2fd7b6-8052-4eb4-a31e-21858150d69e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a2fd7b6-8052-4eb4-a31e-21858150d69e\",\n \"name\": \"5a2fd7b6-8052-4eb4-a31e-21858150d69e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:40.69418327Z\",\n \"endTime\": \"2026-08-25T11:23:43.19279554Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, cluster-image-registry-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00049fd20\u003e: \n failed to create HCP cluster np-downgrade-gwfjf8: failed waiting for cluster=\"np-downgrade-gwfjf8\" in resourcegroup=\"rg-np-downgrade-gwfjf8-7kkgmzwhgxk9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a2fd7b6-8052-4eb4-a31e-21858150d69e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a2fd7b6-8052-4eb4-a31e-21858150d69e\",\n \"name\": \"5a2fd7b6-8052-4eb4-a31e-21858150d69e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:40.69418327Z\",\n \"endTime\": \"2026-08-25T11:23:43.19279554Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, cluster-image-registry-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"660c5eae4bcca0af6b2fe0cfd998feca24db60cd960d79c76beb62534b620fc5","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c4fdcee2df11b558d3b3164768a55e10395e8648b1940e20c553175bae5d91ab","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-t52ww6\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00183c860\u003e: \n failed to create HCP cluster cluster-zstream-4-22-t52ww6: failed waiting for cluster=\"cluster-zstream-4-22-t52ww6\" in resourcegroup=\"rg-zstream-upgrade-4-22-rdnc78mtxx5j\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c696222-49a6-4ca9-b706-8e5a57435480\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c696222-49a6-4ca9-b706-8e5a57435480\",\n \"name\": \"9c696222-49a6-4ca9-b706-8e5a57435480\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.248454811Z\",\n \"endTime\": \"2026-08-25T11:24:07.701920992Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: cluster-image-registry-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-t52ww6\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00183c860\u003e: \n failed to create HCP cluster cluster-zstream-4-22-t52ww6: failed waiting for cluster=\"cluster-zstream-4-22-t52ww6\" in resourcegroup=\"rg-zstream-upgrade-4-22-rdnc78mtxx5j\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c696222-49a6-4ca9-b706-8e5a57435480\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c696222-49a6-4ca9-b706-8e5a57435480\",\n \"name\": \"9c696222-49a6-4ca9-b706-8e5a57435480\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.248454811Z\",\n \"endTime\": \"2026-08-25T11:24:07.701920992Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: cluster-image-registry-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"47245e609d7874f7d25c869a473a1e7e721abc0d258fe07dc19210de937d516d","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"deacd16fd42c584f7532a98cc9b198caf769401900c4d495ba1223ef893fffc3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, cluster-policy-controller, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000936480\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-9ddgb764dmsn\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cd78ff4-590b-4624-b139-ffc7ac943f90\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cd78ff4-590b-4624-b139-ffc7ac943f90\",\n \"name\": \"7cd78ff4-590b-4624-b139-ffc7ac943f90\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.015923741Z\",\n \"endTime\": \"2026-08-25T11:23:43.160352144Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, packageserver, cluster-policy-controller, ingress-operator, olm-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000936480\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-9ddgb764dmsn\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cd78ff4-590b-4624-b139-ffc7ac943f90\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cd78ff4-590b-4624-b139-ffc7ac943f90\",\n \"name\": \"7cd78ff4-590b-4624-b139-ffc7ac943f90\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.015923741Z\",\n \"endTime\": \"2026-08-25T11:23:43.160352144Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, packageserver, cluster-policy-controller, ingress-operator, olm-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"fe7dfc068c8c4065aa24dac07dd321a5e46b5ffb23c7bef9e08da4d256504475","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a169e99d8eb2639f0a1059675f96bc5d6a3645d7a85053031c7e676b4ad42f09","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-j4v9fp\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eaa980\u003e: \n failed to create HCP cluster np-noedge-j4v9fp: failed waiting for cluster=\"np-noedge-j4v9fp\" in resourcegroup=\"rg-np-noedge-j4v9fp-4r7jnknp4pwv\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\",\n \"name\": \"e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:30.442745031Z\",\n \"endTime\": \"2026-08-25T11:23:33.077388395Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-j4v9fp\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eaa980\u003e: \n failed to create HCP cluster np-noedge-j4v9fp: failed waiting for cluster=\"np-noedge-j4v9fp\" in resourcegroup=\"rg-np-noedge-j4v9fp-4r7jnknp4pwv\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\",\n \"name\": \"e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:30.442745031Z\",\n \"endTime\": \"2026-08-25T11:23:33.077388395Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"bbf6b517c402b8dff8d7ee72ef0aee64132aec114ab4d1d13c39df47394f1e2f","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fc48aeaa99b8367ce8df90e24c1324426e971a1da3ed42097c5345639e1ee2c0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, dns-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-p6zdptkz52r2/cluster-candidate-4-21-8fkhjq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e62640\u003e: \n failed to create HCP cluster cluster-candidate-4-21-8fkhjq: failed waiting for cluster=\"cluster-candidate-4-21-8fkhjq\" in resourcegroup=\"rg-candidate-4-21-p6zdptkz52r2\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb1c20d-15c0-4b46-a2f5-9b3caa950874\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb1c20d-15c0-4b46-a2f5-9b3caa950874\",\n \"name\": \"8cb1c20d-15c0-4b46-a2f5-9b3caa950874\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:40.508277835Z\",\n \"endTime\": \"2026-08-25T19:39:42.488198943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, dns-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-p6zdptkz52r2/cluster-candidate-4-21-8fkhjq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e62640\u003e: \n failed to create HCP cluster cluster-candidate-4-21-8fkhjq: failed waiting for cluster=\"cluster-candidate-4-21-8fkhjq\" in resourcegroup=\"rg-candidate-4-21-p6zdptkz52r2\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb1c20d-15c0-4b46-a2f5-9b3caa950874\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb1c20d-15c0-4b46-a2f5-9b3caa950874\",\n \"name\": \"8cb1c20d-15c0-4b46-a2f5-9b3caa950874\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:40.508277835Z\",\n \"endTime\": \"2026-08-25T19:39:42.488198943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, dns-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"32b84312ecc0412cc4b93a9ef15c32fc382121719b6562881ec5a616688de541","pr_number":6683,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a0371ff83ff8ae672d5d38be2f67ae6f726e275d3db1d866178e427376320fd8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011325d0\u003e: \n GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/642ab867-7117-49d1-9205-fcc81b8fcac8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/642ab867-7117-49d1-9205-fcc81b8fcac8\",\n \"name\": \"642ab867-7117-49d1-9205-fcc81b8fcac8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:23.530391484Z\",\n \"endTime\": \"2026-08-25T11:22:27.900506106Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011325d0\u003e: \n GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/642ab867-7117-49d1-9205-fcc81b8fcac8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/642ab867-7117-49d1-9205-fcc81b8fcac8\",\n \"name\": \"642ab867-7117-49d1-9205-fcc81b8fcac8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:23.530391484Z\",\n \"endTime\": \"2026-08-25T11:22:27.900506106Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"f83b12586d201faafc5d77e61170761a01913b4aa6e23e7374f34e7086cab7fc","pr_number":6670,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fc2f22952e29415c0ffa879b795fd94133860d84ce001832877fc2b42797fd71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048a80\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-znbww55lj6vk\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32f30388-3946-451c-abc4-89ec972b44f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32f30388-3946-451c-abc4-89ec972b44f5\",\n \"name\": \"32f30388-3946-451c-abc4-89ec972b44f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:32.564773496Z\",\n \"endTime\": \"2026-08-25T11:21:36.023308749Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-network-operator, openshift-controller-manager, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048a80\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-znbww55lj6vk\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32f30388-3946-451c-abc4-89ec972b44f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32f30388-3946-451c-abc4-89ec972b44f5\",\n \"name\": \"32f30388-3946-451c-abc4-89ec972b44f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:32.564773496Z\",\n \"endTime\": \"2026-08-25T11:21:36.023308749Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-network-operator, openshift-controller-manager, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"59617915775eae13a40addb69d6b99b7fe7f688ae59ce79fcd656898578c9492","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"910e6132d71017042ccb4ad8536b725ec097c3baf39c134fb4939471b7ea12a4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001696280\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-j578jj9q95nl\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd6036ee-637e-4111-981e-76356c491fc6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd6036ee-637e-4111-981e-76356c491fc6\",\n \"name\": \"dd6036ee-637e-4111-981e-76356c491fc6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.42538851Z\",\n \"endTime\": \"2026-08-25T11:23:43.113756051Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, ingress-operator, olm-operator, catalog-operator, packageserver, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001696280\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-j578jj9q95nl\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd6036ee-637e-4111-981e-76356c491fc6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd6036ee-637e-4111-981e-76356c491fc6\",\n \"name\": \"dd6036ee-637e-4111-981e-76356c491fc6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.42538851Z\",\n \"endTime\": \"2026-08-25T11:23:43.113756051Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, ingress-operator, olm-operator, catalog-operator, packageserver, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"9a4f67edde370308f7181e7ac96b3504189ee39d35011561ed75c7aba37ebe2d","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5f2092ac8b60125a2e24e70258e522230cf6619c6ea4fc2f4a10c6a15fc6e2e8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e24460\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-cmpnvt9r42hk\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f79a299e-0c13-4f75-b94f-813c143c5081\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f79a299e-0c13-4f75-b94f-813c143c5081\",\n \"name\": \"f79a299e-0c13-4f75-b94f-813c143c5081\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:09.116151219Z\",\n \"endTime\": \"2026-08-25T11:17:17.145229508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e24460\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-cmpnvt9r42hk\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f79a299e-0c13-4f75-b94f-813c143c5081\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f79a299e-0c13-4f75-b94f-813c143c5081\",\n \"name\": \"f79a299e-0c13-4f75-b94f-813c143c5081\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:09.116151219Z\",\n \"endTime\": \"2026-08-25T11:17:17.145229508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"a557455b2f15d5267c35100305fbb366e3ce937bce05ea40aa0c0c92aaacdae2","pr_number":6648,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3c03ff8c6f57d65dc5b13024ef220ab850aed54bb6437f3a0bfad2241929fe27","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator, cluster-policy-controller, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-g4wtj9 with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8a620\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-g4wtj9: failed waiting for cluster=\"np-version-upgrade-cluster-g4wtj9\" in resourcegroup=\"rg-np-version-upgrade-g4wtj9-mrccxbqqnhhc\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d40e3df-681f-4926-943f-226d7903f59e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d40e3df-681f-4926-943f-226d7903f59e\",\n \"name\": \"7d40e3df-681f-4926-943f-226d7903f59e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:40.789915832Z\",\n \"endTime\": \"2026-08-25T11:23:43.181882794Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-policy-controller, cluster-node-tuning-operator, azure-cloud-controller-manager, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-g4wtj9 with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8a620\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-g4wtj9: failed waiting for cluster=\"np-version-upgrade-cluster-g4wtj9\" in resourcegroup=\"rg-np-version-upgrade-g4wtj9-mrccxbqqnhhc\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d40e3df-681f-4926-943f-226d7903f59e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d40e3df-681f-4926-943f-226d7903f59e\",\n \"name\": \"7d40e3df-681f-4926-943f-226d7903f59e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:40.789915832Z\",\n \"endTime\": \"2026-08-25T11:23:43.181882794Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-policy-controller, cluster-node-tuning-operator, azure-cloud-controller-manager, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"19ac66d7406c2cf544c19dab33599323ce924d5b29029bc616b279fbe8796bf1","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"55a75f995426be3c05299ad4947251679d4a99ba7e77d726aeadaba9bd31ae5d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, kube-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-kp6cj8gslgn7/cluster-candidate-4-20-2ftmsx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101ebe0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-2ftmsx: failed waiting for cluster=\"cluster-candidate-4-20-2ftmsx\" in resourcegroup=\"rg-candidate-4-20-kp6cj8gslgn7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ce27661-0cfc-4c0b-a2fe-99358f1c893d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ce27661-0cfc-4c0b-a2fe-99358f1c893d\",\n \"name\": \"9ce27661-0cfc-4c0b-a2fe-99358f1c893d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.525443298Z\",\n \"endTime\": \"2026-08-25T11:24:07.636192222Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-kp6cj8gslgn7/cluster-candidate-4-20-2ftmsx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101ebe0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-2ftmsx: failed waiting for cluster=\"cluster-candidate-4-20-2ftmsx\" in resourcegroup=\"rg-candidate-4-20-kp6cj8gslgn7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ce27661-0cfc-4c0b-a2fe-99358f1c893d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ce27661-0cfc-4c0b-a2fe-99358f1c893d\",\n \"name\": \"9ce27661-0cfc-4c0b-a2fe-99358f1c893d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.525443298Z\",\n \"endTime\": \"2026-08-25T11:24:07.636192222Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"7311a955c2510af435a3dd4e8bda65a101a0cfae9846f5be659992b940243062","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"75a2ddc5380cf22cdc0f7eee734c0875c6030bb3df880d9c39c49bb360b07d00","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator, ignition-server-proxy, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a56680\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-h8x2lnscjddq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/306db8ae-e454-406c-b2d9-5d6ef9785b20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/306db8ae-e454-406c-b2d9-5d6ef9785b20\",\n \"name\": \"306db8ae-e454-406c-b2d9-5d6ef9785b20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:01.374469415Z\",\n \"endTime\": \"2026-08-25T11:24:07.731322209Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a56680\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-h8x2lnscjddq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/306db8ae-e454-406c-b2d9-5d6ef9785b20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/306db8ae-e454-406c-b2d9-5d6ef9785b20\",\n \"name\": \"306db8ae-e454-406c-b2d9-5d6ef9785b20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:01.374469415Z\",\n \"endTime\": \"2026-08-25T11:24:07.731322209Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"674600a738c7667770e739f9055e1d79ea50c97b5b16436d10f3b9df4e0297d0","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b67b2954ed80e23d5dbe844cbe69ddf2aee82b77fbb4d9b923a377beae494cff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e7a340\u003e: \n failed to create HCP cluster np-dg-minor-5k4t7m: failed waiting for cluster=\"np-dg-minor-5k4t7m\" in resourcegroup=\"rg-np-dg-minor-5k4t7m-c7rmk8jp25j5\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36c411c9-a482-476f-8c6d-e111f54a5519\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36c411c9-a482-476f-8c6d-e111f54a5519\",\n \"name\": \"36c411c9-a482-476f-8c6d-e111f54a5519\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.5276884Z\",\n \"endTime\": \"2026-08-25T11:21:35.939284741Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e7a340\u003e: \n failed to create HCP cluster np-dg-minor-5k4t7m: failed waiting for cluster=\"np-dg-minor-5k4t7m\" in resourcegroup=\"rg-np-dg-minor-5k4t7m-c7rmk8jp25j5\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36c411c9-a482-476f-8c6d-e111f54a5519\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36c411c9-a482-476f-8c6d-e111f54a5519\",\n \"name\": \"36c411c9-a482-476f-8c6d-e111f54a5519\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.5276884Z\",\n \"endTime\": \"2026-08-25T11:21:35.939284741Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"4b8149b33710d3d2d55931ca7538e774a86bcfc7ef3d236c36f5ffb2ecb5845f","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"01b6bec7cc22825d4ef87b95818e501dbd62c6d195e057bf40cb1fb4d00f77e0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013da220\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-hk9dqp59j9m5\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\",\n \"name\": \"cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:54.161881545Z\",\n \"endTime\": \"2026-08-25T11:20:57.408325813Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013da220\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-hk9dqp59j9m5\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\",\n \"name\": \"cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:54.161881545Z\",\n \"endTime\": \"2026-08-25T11:20:57.408325813Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"dbfac9a592c7d3b659769802f66b5c2ea426b38b8a462e72863bc31b516fbcfb","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"daeefc59f58572b03a630f3ab57fcec5e15bddea741d2154444f66cb96ddf3d9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, dns-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001a5060\u003e: \n failed to create HCP cluster np-dg-minor-gzvdzt: failed waiting for cluster=\"np-dg-minor-gzvdzt\" in resourcegroup=\"rg-np-dg-minor-gzvdzt-zrllgdp7qt99\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b036348-ef9c-491d-be11-832d77741098\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b036348-ef9c-491d-be11-832d77741098\",\n \"name\": \"5b036348-ef9c-491d-be11-832d77741098\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:00.551807818Z\",\n \"endTime\": \"2026-08-25T14:40:03.659175976Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001a5060\u003e: \n failed to create HCP cluster np-dg-minor-gzvdzt: failed waiting for cluster=\"np-dg-minor-gzvdzt\" in resourcegroup=\"rg-np-dg-minor-gzvdzt-zrllgdp7qt99\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b036348-ef9c-491d-be11-832d77741098\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b036348-ef9c-491d-be11-832d77741098\",\n \"name\": \"5b036348-ef9c-491d-be11-832d77741098\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:00.551807818Z\",\n \"endTime\": \"2026-08-25T14:40:03.659175976Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"268457b3abac70cef4adc34ed78ed5e91defa790eb3f46b44d42d7bf24a4ab3d","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"edc445226bca8c15a6801b9e4e1854b6c7f899e847f419d892bdf3d1968a9d97","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator, dns-operator, ignition-server-proxy, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-n44w2df8vj8t/cluster-candidate-4-21-mppmvd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c4140\u003e: \n failed to create HCP cluster cluster-candidate-4-21-mppmvd: failed waiting for cluster=\"cluster-candidate-4-21-mppmvd\" in resourcegroup=\"rg-candidate-4-21-n44w2df8vj8t\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\",\n \"name\": \"5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.19951964Z\",\n \"endTime\": \"2026-08-25T11:20:57.290621961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-n44w2df8vj8t/cluster-candidate-4-21-mppmvd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c4140\u003e: \n failed to create HCP cluster cluster-candidate-4-21-mppmvd: failed waiting for cluster=\"cluster-candidate-4-21-mppmvd\" in resourcegroup=\"rg-candidate-4-21-n44w2df8vj8t\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\",\n \"name\": \"5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.19951964Z\",\n \"endTime\": \"2026-08-25T11:20:57.290621961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"d5ffa576330ff4e403d279cd3e6697f2671ddbf3167fa412a03694b6d7d28267","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"55058ba88f617860ddd7645f53374b164637eda1467294e44a5e2bd25e9411f0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [dns-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000edefa0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rxw4z4c4mjfj\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7f39717-bb95-4ba8-b531-5600d9435095\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7f39717-bb95-4ba8-b531-5600d9435095\",\n \"name\": \"d7f39717-bb95-4ba8-b531-5600d9435095\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:37.648275001Z\",\n \"endTime\": \"2026-08-25T11:22:43.091522188Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000edefa0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rxw4z4c4mjfj\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7f39717-bb95-4ba8-b531-5600d9435095\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7f39717-bb95-4ba8-b531-5600d9435095\",\n \"name\": \"d7f39717-bb95-4ba8-b531-5600d9435095\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:37.648275001Z\",\n \"endTime\": \"2026-08-25T11:22:43.091522188Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"1c7a8ccc960575f74b5f63b9d0fee47fa26ccbebde3ae4de7a7bbd10588a7ab3","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8fcfad7ee636c64bcc7ec16c2a0da480eb90b3d648cf1239ba480534695bb74a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010003c0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-dgghk7xd8fkc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/813de9e6-c926-4fa8-a38a-500226e3059a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/813de9e6-c926-4fa8-a38a-500226e3059a\",\n \"name\": \"813de9e6-c926-4fa8-a38a-500226e3059a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.708804056Z\",\n \"endTime\": \"2026-08-25T11:24:07.512424839Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, hosted-cluster-config-operator, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010003c0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-dgghk7xd8fkc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/813de9e6-c926-4fa8-a38a-500226e3059a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/813de9e6-c926-4fa8-a38a-500226e3059a\",\n \"name\": \"813de9e6-c926-4fa8-a38a-500226e3059a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.708804056Z\",\n \"endTime\": \"2026-08-25T11:24:07.512424839Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, hosted-cluster-config-operator, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"18f9ebc5eb73ff84a02d71ec227c141bf94f7ba02eeea0d33eed11a56429d19b","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"691000237ed68ee2bea1eca4972f51b642a5c69ce54dd17b387ef5ef88b01453","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d02900\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-sn9l5gh5sgnq\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77d8a5d2-c15f-4303-9203-ed2a4dc77c20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77d8a5d2-c15f-4303-9203-ed2a4dc77c20\",\n \"name\": \"77d8a5d2-c15f-4303-9203-ed2a4dc77c20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:02.408692636Z\",\n \"endTime\": \"2026-08-25T11:21:07.333560396Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d02900\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-sn9l5gh5sgnq\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77d8a5d2-c15f-4303-9203-ed2a4dc77c20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77d8a5d2-c15f-4303-9203-ed2a4dc77c20\",\n \"name\": \"77d8a5d2-c15f-4303-9203-ed2a4dc77c20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:02.408692636Z\",\n \"endTime\": \"2026-08-25T11:21:07.333560396Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"c5290fb9c57d434bec526c46d513016dd6789c9d085ab04d2a242efeb095b470","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"183ac5e7e1c7f43ad252fd54e6e3fae5ee8421fc47486bdad5d091cff0dbf9a9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator, openshift-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-qpdbsl with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e015c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-qpdbsl: failed waiting for cluster=\"np-version-upgrade-cluster-qpdbsl\" in resourcegroup=\"rg-np-version-upgrade-qpdbsl-xxx55fsnfv5d\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\",\n \"name\": \"35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.081081979Z\",\n \"endTime\": \"2026-08-25T11:24:07.516329781Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-qpdbsl with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e015c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-qpdbsl: failed waiting for cluster=\"np-version-upgrade-cluster-qpdbsl\" in resourcegroup=\"rg-np-version-upgrade-qpdbsl-xxx55fsnfv5d\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\",\n \"name\": \"35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.081081979Z\",\n \"endTime\": \"2026-08-25T11:24:07.516329781Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"6f6befa657f715c3b4b4b66b7a559071e94c36c338612799fd64e9fc60a3ceb0","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2256dab0b9b860af42d8774bc9e3bdf3a75814dca1080b715190d3d8ab7a2eb9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706220\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-tdgbq5vc5q77\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\",\n \"name\": \"e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:45.146044639Z\",\n \"endTime\": \"2026-08-25T11:21:47.576163129Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, ingress-operator; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706220\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-tdgbq5vc5q77\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\",\n \"name\": \"e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:45.146044639Z\",\n \"endTime\": \"2026-08-25T11:21:47.576163129Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, ingress-operator; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"80ba9276ccee66ce5f15246d1c46ed27a227c963837a0639556a2c98e2273b4b","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5d1725423ee4b212bccb0d13858a78dd47769db5d794223ea2e26d6df7c4f5ad","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-rc5jrpz77lgk/cluster-candidate-5-0-k8xlxd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001060a40\u003e: \n failed to create HCP cluster cluster-candidate-5-0-k8xlxd: failed waiting for cluster=\"cluster-candidate-5-0-k8xlxd\" in resourcegroup=\"rg-candidate-5-0-rc5jrpz77lgk\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/223cfa5a-75b0-4cb9-b839-dc5100d41095\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/223cfa5a-75b0-4cb9-b839-dc5100d41095\",\n \"name\": \"223cfa5a-75b0-4cb9-b839-dc5100d41095\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:32.648244013Z\",\n \"endTime\": \"2026-08-25T11:20:37.341589368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-rc5jrpz77lgk/cluster-candidate-5-0-k8xlxd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001060a40\u003e: \n failed to create HCP cluster cluster-candidate-5-0-k8xlxd: failed waiting for cluster=\"cluster-candidate-5-0-k8xlxd\" in resourcegroup=\"rg-candidate-5-0-rc5jrpz77lgk\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/223cfa5a-75b0-4cb9-b839-dc5100d41095\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/223cfa5a-75b0-4cb9-b839-dc5100d41095\",\n \"name\": \"223cfa5a-75b0-4cb9-b839-dc5100d41095\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:32.648244013Z\",\n \"endTime\": \"2026-08-25T11:20:37.341589368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"3ab9e7c7d4f2d79e65f9b8a56c90ee66becb35144ac46e78ca458d7f63faa98d","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bb52007d345fecf08b7c3e5921187b32bee560cfda2f40a385c2e2262d097a34","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a1cce0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-994pfg4rbw2z\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\",\n \"name\": \"f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:18.051164248Z\",\n \"endTime\": \"2026-08-25T11:23:23.080049738Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, konnectivity-agent, catalog-operator, control-plane-pki-operator, openshift-apiserver, ignition-server-proxy, openshift-route-controller-manager, cluster-autoscaler, machine-approver, kube-controller-manager, azure-cloud-controller-manager, redhat-marketplace-catalog, openshift-controller-manager, hosted-cluster-config-operator, olm-operator, cluster-version-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, cluster-image-registry-operator, olm-collect-profiles, certified-operators-catalog, cluster-node-tuning-operator, ingress-operator, packageserver, redhat-operators-catalog, community-operators-catalog, kube-scheduler, ignition-server, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a1cce0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-994pfg4rbw2z\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\",\n \"name\": \"f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:18.051164248Z\",\n \"endTime\": \"2026-08-25T11:23:23.080049738Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, konnectivity-agent, catalog-operator, control-plane-pki-operator, openshift-apiserver, ignition-server-proxy, openshift-route-controller-manager, cluster-autoscaler, machine-approver, kube-controller-manager, azure-cloud-controller-manager, redhat-marketplace-catalog, openshift-controller-manager, hosted-cluster-config-operator, olm-operator, cluster-version-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, cluster-image-registry-operator, olm-collect-profiles, certified-operators-catalog, cluster-node-tuning-operator, ingress-operator, packageserver, redhat-operators-catalog, community-operators-catalog, kube-scheduler, ignition-server, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"1ccba13814040aca0b43b4df7ff1901195302261df9120296071d4873aa80760","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4b3c5c6b82d824a461fdf0a9ceb99cba2cdf827bff592ead991249815b65b39a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001350960\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-44brmcrqlwhd\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc250653-c8e3-42dd-bdb2-80628cccd921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc250653-c8e3-42dd-bdb2-80628cccd921\",\n \"name\": \"bc250653-c8e3-42dd-bdb2-80628cccd921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:24:55.65138247Z\",\n \"endTime\": \"2026-08-25T16:43:58.821172119Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, redhat-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001350960\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-44brmcrqlwhd\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc250653-c8e3-42dd-bdb2-80628cccd921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc250653-c8e3-42dd-bdb2-80628cccd921\",\n \"name\": \"bc250653-c8e3-42dd-bdb2-80628cccd921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:24:55.65138247Z\",\n \"endTime\": \"2026-08-25T16:43:58.821172119Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, redhat-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","signature_id":"a997df37747bd9ae4adc6c1d01f9efa7c9eeb6284eec6b1bc5d69624dc28ac10","pr_number":4566,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f1744105155154513ebbf4bd7117f8888fd7e8a5f221414547d7257b1ee3bbdd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e26600\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-g94rgnmwffff\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00f455ba-7a0f-4042-81b1-6b4e1d717422\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00f455ba-7a0f-4042-81b1-6b4e1d717422\",\n \"name\": \"00f455ba-7a0f-4042-81b1-6b4e1d717422\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.361104Z\",\n \"endTime\": \"2026-08-25T11:21:33.042291432Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e26600\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-g94rgnmwffff\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00f455ba-7a0f-4042-81b1-6b4e1d717422\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00f455ba-7a0f-4042-81b1-6b4e1d717422\",\n \"name\": \"00f455ba-7a0f-4042-81b1-6b4e1d717422\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.361104Z\",\n \"endTime\": \"2026-08-25T11:21:33.042291432Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"8306a5db6f3a5a75eed1ac89929790d5c6dfaf67eb9701d7ccbd095c361aa3a6","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f59d41db8c7110a2a66df7a04a929104d4f74b17a121064979556d85ac966c14","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000976d60\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sp25f888spq4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3198a15-7882-4985-8fb9-601f276da5ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3198a15-7882-4985-8fb9-601f276da5ee\",\n \"name\": \"d3198a15-7882-4985-8fb9-601f276da5ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:58.802251942Z\",\n \"endTime\": \"2026-08-25T11:24:07.715404645Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000976d60\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sp25f888spq4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3198a15-7882-4985-8fb9-601f276da5ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3198a15-7882-4985-8fb9-601f276da5ee\",\n \"name\": \"d3198a15-7882-4985-8fb9-601f276da5ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:58.802251942Z\",\n \"endTime\": \"2026-08-25T11:24:07.715404645Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"4bc6fa9b62fea5fb0a886bde62212833e54bd33c1122e0a8839c522e31ede058","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f7ad14e1158d7d53f17395adc1367d8f6b3116098187de178cccafef487d5b29","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f5ae80\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-jpmgv9dccmdw\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ce80b32-21f8-4a00-a1a0-eb828a65f28d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ce80b32-21f8-4a00-a1a0-eb828a65f28d\",\n \"name\": \"0ce80b32-21f8-4a00-a1a0-eb828a65f28d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:45.150172423Z\",\n \"endTime\": \"2026-08-25T11:23:47.559518518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f5ae80\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-jpmgv9dccmdw\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ce80b32-21f8-4a00-a1a0-eb828a65f28d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ce80b32-21f8-4a00-a1a0-eb828a65f28d\",\n \"name\": \"0ce80b32-21f8-4a00-a1a0-eb828a65f28d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:45.150172423Z\",\n \"endTime\": \"2026-08-25T11:23:47.559518518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"7c0e987f55407504abfc17fba96a678154c2fb6eca07c78b46a987ce3627cfc1","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8a4546d1ccb11eb2bd205f13e52f18ff56616b802ad979a8ada123b2e8ac5a07","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d08560\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mx5c2xs4fbj7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0554f1ec-f6c0-4f0c-869c-8157fe13f06e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0554f1ec-f6c0-4f0c-869c-8157fe13f06e\",\n \"name\": \"0554f1ec-f6c0-4f0c-869c-8157fe13f06e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:26.179895607Z\",\n \"endTime\": \"2026-08-25T11:25:27.51902962Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d08560\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mx5c2xs4fbj7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0554f1ec-f6c0-4f0c-869c-8157fe13f06e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0554f1ec-f6c0-4f0c-869c-8157fe13f06e\",\n \"name\": \"0554f1ec-f6c0-4f0c-869c-8157fe13f06e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:26.179895607Z\",\n \"endTime\": \"2026-08-25T11:25:27.51902962Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"e7d58e0615757ed6b9924d8c23510714583aa7bdf89e2194c35463a6520b17a5","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9917e0e4cec099cb18cbbcab5a48ffe13a11b43e10f38323888849ed9a27dd49","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, openshift-route-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00058a120\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-hbkfwxhrsvb2\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ad145b-c45f-488c-9b63-9a804b8af221\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ad145b-c45f-488c-9b63-9a804b8af221\",\n \"name\": \"51ad145b-c45f-488c-9b63-9a804b8af221\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:15.619518812Z\",\n \"endTime\": \"2026-08-25T11:24:17.933561304Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00058a120\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-hbkfwxhrsvb2\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ad145b-c45f-488c-9b63-9a804b8af221\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ad145b-c45f-488c-9b63-9a804b8af221\",\n \"name\": \"51ad145b-c45f-488c-9b63-9a804b8af221\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:15.619518812Z\",\n \"endTime\": \"2026-08-25T11:24:17.933561304Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"d6a56eb98a1093f3a5c49d3962df7ba45a52a81222c930c3ba06aff802dd1a1f","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"442531230d6bcd61fe689ddb6dd1199a972aefc3d6bea05b5c34201c97dcd098","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001336360\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4mq79tbwfj5b\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a3c034-245f-49ad-8068-0b454f6ec56c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a3c034-245f-49ad-8068-0b454f6ec56c\",\n \"name\": \"a6a3c034-245f-49ad-8068-0b454f6ec56c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:48.128240196Z\",\n \"endTime\": \"2026-08-25T19:37:52.295117327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001336360\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4mq79tbwfj5b\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a3c034-245f-49ad-8068-0b454f6ec56c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a3c034-245f-49ad-8068-0b454f6ec56c\",\n \"name\": \"a6a3c034-245f-49ad-8068-0b454f6ec56c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:48.128240196Z\",\n \"endTime\": \"2026-08-25T19:37:52.295117327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"bc247a67bb8f719d23e560a19b4f04763bcfe81ecbf03218468d9c66f357d587","pr_number":6683,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b652815635f16d983d1325dbd7e304560647aed7cf1be6254e8a2a57faf1d4fa","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef90e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2zg88lxq9q4w\" to finish creating: GET https://rp.j3558656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\",\n \"name\": \"b976f9f3-2891-4433-868a-76a8d23de317\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T16:54:08.369328068Z\",\n \"endTime\": \"2026-08-24T17:13:08.208412685Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef90e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2zg88lxq9q4w\" to finish creating: GET https://rp.j3558656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\",\n \"name\": \"b976f9f3-2891-4433-868a-76a8d23de317\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T16:54:08.369328068Z\",\n \"endTime\": \"2026-08-24T17:13:08.208412685Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091920407983558656","occurred_at":"2026-08-24T16:06:36Z","signature_id":"fdde03128e739a2f562ab5c3485dae63a0f561b3b4659197733a62ff1ea2a4e8","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bebd56cb7971d5cbd2ba0627c63fe9925ffa3a6799fdb1d98a37f3159a2c38aa","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097b680\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wgz7h9k2skjd\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a299a2cd-ef58-4c09-8fa3-3315683149e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a299a2cd-ef58-4c09-8fa3-3315683149e1\",\n \"name\": \"a299a2cd-ef58-4c09-8fa3-3315683149e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.19619894Z\",\n \"endTime\": \"2026-08-25T11:20:57.271055091Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097b680\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wgz7h9k2skjd\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a299a2cd-ef58-4c09-8fa3-3315683149e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a299a2cd-ef58-4c09-8fa3-3315683149e1\",\n \"name\": \"a299a2cd-ef58-4c09-8fa3-3315683149e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.19619894Z\",\n \"endTime\": \"2026-08-25T11:20:57.271055091Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"f45e48a3b1e3da0b3b86c68ae16171bf54eb097156d7da3966f4506df7b9b4dc","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b460f0e72ca79f88845e6e8bf228b8d0be2539d0f8bce034dfbee5009ef88258","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3e1a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nlgtzd8jj8ls\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c6c05ac-d3c3-4c18-b01d-729aef0c992e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c6c05ac-d3c3-4c18-b01d-729aef0c992e\",\n \"name\": \"5c6c05ac-d3c3-4c18-b01d-729aef0c992e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.862453094Z\",\n \"endTime\": \"2026-08-25T11:21:35.935998302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3e1a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nlgtzd8jj8ls\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c6c05ac-d3c3-4c18-b01d-729aef0c992e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c6c05ac-d3c3-4c18-b01d-729aef0c992e\",\n \"name\": \"5c6c05ac-d3c3-4c18-b01d-729aef0c992e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.862453094Z\",\n \"endTime\": \"2026-08-25T11:21:35.935998302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"bc8afc40785a7de130f0afb33506490a59b6681bd8c5fec932af8efd346aaf8c","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"09f986eeaef9ef03e89acc8ef57bf3aa4f7820de8da84049b2bc414381ff7c7f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee3660\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pfd8fc762xff\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9934e2d-ec51-45a3-8cbd-60f4853ece87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9934e2d-ec51-45a3-8cbd-60f4853ece87\",\n \"name\": \"c9934e2d-ec51-45a3-8cbd-60f4853ece87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:47:46.321950941Z\",\n \"endTime\": \"2026-08-25T15:06:53.075223567Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee3660\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pfd8fc762xff\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9934e2d-ec51-45a3-8cbd-60f4853ece87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9934e2d-ec51-45a3-8cbd-60f4853ece87\",\n \"name\": \"c9934e2d-ec51-45a3-8cbd-60f4853ece87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:47:46.321950941Z\",\n \"endTime\": \"2026-08-25T15:06:53.075223567Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"279d843775eb67f9cc4f7628611a62e9cdb3df14488565ed034ba6ce68dad623","pr_number":6630,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"94457f237fe77f556896b3c5dcbbf4f6fa7202d5ac7c0a13c7473270227156e1","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea0340\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-cm65c9f47lvs\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\",\n \"name\": \"87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:42:45.179574291Z\",\n \"endTime\": \"2026-08-25T17:01:49.211726949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea0340\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-cm65c9f47lvs\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\",\n \"name\": \"87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:42:45.179574291Z\",\n \"endTime\": \"2026-08-25T17:01:49.211726949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"3b0400f6feabfc5e3f875e41b2ad13cd2a30893f4a99364d4e6f721f7666dfca","pr_number":6679,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8172b3b938ddc92e532decd443ed82e33579c57cbdab74ca3567337de4aab992","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d81e60\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-sx5z6bzj9gt4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9107795d-3348-494d-85dd-e9b48aaf131e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9107795d-3348-494d-85dd-e9b48aaf131e\",\n \"name\": \"9107795d-3348-494d-85dd-e9b48aaf131e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:17.129504958Z\",\n \"endTime\": \"2026-08-25T11:24:17.573823206Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d81e60\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-sx5z6bzj9gt4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9107795d-3348-494d-85dd-e9b48aaf131e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9107795d-3348-494d-85dd-e9b48aaf131e\",\n \"name\": \"9107795d-3348-494d-85dd-e9b48aaf131e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:17.129504958Z\",\n \"endTime\": \"2026-08-25T11:24:17.573823206Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"76e7fe8a79658b5944b75f155ea6ce85c0f606666b8630971299524820b46509","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-cbhfzqtvb9th/cluster-candidate-4-21-5bbfxw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea2680\u003e: \n failed to create HCP cluster cluster-candidate-4-21-5bbfxw: failed waiting for cluster=\"cluster-candidate-4-21-5bbfxw\" in resourcegroup=\"rg-candidate-4-21-cbhfzqtvb9th\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n \"name\": \"a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:32.844651935Z\",\n \"endTime\": \"2026-08-24T17:47:34.560561952Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-cbhfzqtvb9th/cluster-candidate-4-21-5bbfxw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea2680\u003e: \n failed to create HCP cluster cluster-candidate-4-21-5bbfxw: failed waiting for cluster=\"cluster-candidate-4-21-5bbfxw\" in resourcegroup=\"rg-candidate-4-21-cbhfzqtvb9th\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n \"name\": \"a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:32.844651935Z\",\n \"endTime\": \"2026-08-24T17:47:34.560561952Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"94e14a4948dabf46772c6e049c6c316329369f07279d585a70084b3a5b07d8b8","pr_number":6657,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bad17b00fd2861c199fb8c7332ad5eb58ff1eb35d64efc7066c892df3da78afa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] \u003cno_message\u003e; [clusterServiceStatus] \u003cno_message\u003e; [descendantResources] \u003cno_message\u003e; [hostedCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":7,"also_in":["STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":19,"prior_last_seen_at":"2026-08-21T04:29:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001c982a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-gdnhnw\" in resourcegroup=\"rg-hs-pre--hm7prkdxbkw6\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\",\n \"name\": \"67ed1acc-0077-4b07-848d-484f46170f44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:21:50.42303027Z\",\n \"endTime\": \"2026-08-24T15:45:59.872804145Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001c982a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-gdnhnw\" in resourcegroup=\"rg-hs-pre--hm7prkdxbkw6\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\",\n \"name\": \"67ed1acc-0077-4b07-848d-484f46170f44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:21:50.42303027Z\",\n \"endTime\": \"2026-08-24T15:45:59.872804145Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"cf9ddb524cae2c0c6a37a93000fda2cd14fe56bff50daec6b5b3ed77ad21467c","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":117,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":125,"prior_last_seen_at":"2026-08-22T00:13:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0026c6318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-krw8vkxmtzpb\" to finish deleting: GET https://rp.j5592064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74596e16-42cf-45f7-b5d0-faa23e1127f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74596e16-42cf-45f7-b5d0-faa23e1127f5\",\n \"name\": \"74596e16-42cf-45f7-b5d0-faa23e1127f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:56:49.253326502Z\",\n \"endTime\": \"2026-08-25T11:20:52.069849414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdi5csg0pn2q641bbc4hk7ubn3ksvmp still exists (deletion dispatched at 2026-08-25T10:56:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0026c6318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-krw8vkxmtzpb\" to finish deleting: GET https://rp.j5592064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74596e16-42cf-45f7-b5d0-faa23e1127f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74596e16-42cf-45f7-b5d0-faa23e1127f5\",\n \"name\": \"74596e16-42cf-45f7-b5d0-faa23e1127f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:56:49.253326502Z\",\n \"endTime\": \"2026-08-25T11:20:52.069849414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdi5csg0pn2q641bbc4hk7ubn3ksvmp still exists (deletion dispatched at 2026-08-25T10:56:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092189876085592064","occurred_at":"2026-08-25T09:52:51Z","signature_id":"4dcc3e0028de7e96928b0a1a88758b6efab2ee551db9968cdbece8675c668057","pr_number":6669,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ba89a4166305f6c19c8eaadc8056c4375814afedeefcd4ac9c286060c7d5cad1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-10"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-14T20:23:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000ebc2a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-h4ldj6\" in resourcegroup=\"rg-candidate-5-0-xjnldggtrt7m\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\",\n \"name\": \"fa0102ea-068c-4907-92f2-69579c623cac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:21:51.698922375Z\",\n \"endTime\": \"2026-08-24T15:45:59.987785659Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm5ibpilvjmdnr77hbqf9t255bon still exists (deletion dispatched at 2026-08-24T15:21:51Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000ebc2a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-h4ldj6\" in resourcegroup=\"rg-candidate-5-0-xjnldggtrt7m\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\",\n \"name\": \"fa0102ea-068c-4907-92f2-69579c623cac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:21:51.698922375Z\",\n \"endTime\": \"2026-08-24T15:45:59.987785659Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm5ibpilvjmdnr77hbqf9t255bon still exists (deletion dispatched at 2026-08-24T15:21:51Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"60c407c2c17844ded754b0d5e8dda40f125391008bbbdd669c23fa4348b5b316","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-22T00:26:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0010deff0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-4cmmkc\" in resourcegroup=\"admin-credential-lifecycle-test-mk5bt6ghv29k\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\",\n \"name\": \"0644f278-df05-4603-a10d-3a66a1f851d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:17:03.672410989Z\",\n \"endTime\": \"2026-08-24T15:41:10.046088761Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tjfmep0gog9fd4pfkr5bnpc0n642 still exists (deletion dispatched at 2026-08-24T15:17:04Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0010deff0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-4cmmkc\" in resourcegroup=\"admin-credential-lifecycle-test-mk5bt6ghv29k\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\",\n \"name\": \"0644f278-df05-4603-a10d-3a66a1f851d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:17:03.672410989Z\",\n \"endTime\": \"2026-08-24T15:41:10.046088761Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tjfmep0gog9fd4pfkr5bnpc0n642 still exists (deletion dispatched at 2026-08-24T15:17:04Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"7df46629584e4c4253c91624b483f13865f970598c9ed547ff855ac9eea68fe2","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1a135150bda94b29bbfd120583fd4db7aacc9a2d400307335c08f2083d06f63a","failure_pattern":"ERROR: HTTPSConnectionPool(host='login.microsoftonline.com', port=443): Max retries exceeded with url: /XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/oauth2/v2.0/token (Caused by NameResolutionError(\"HTTPSConnection(host='login.microsoftonline.com', port=443): Failed","search_query":"ERROR: HTTPSConnectionPool(host='login.microsoftonline.com', port=443): Max retries exceeded with url: /XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/oauth2/v2.0/token (Caused by NameResolutionError(\\\"HTTPSConnection(host='login.microsoftonline.com', port=443): Failed to resolve 'login.microsoftonline.com' ([Errno -2] Name or service not known)\\\"))","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/global/mirror-image","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/global/mirror-image","occurrences":1}],"full_error_samples":["time=2026-08-25T05:18:44.021Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e maestro.image.registry \u003cnil\u003e}:{\u003cnil\u003e maestro.image.repository \u003cnil\u003e}@{\u003cnil\u003e maestro.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-08-25T05:19:25.373Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nERROR: HTTPSConnectionPool(host='login.microsoftonline.com', port=443): Max retries exceeded with url: /XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/oauth2/v2.0/token (Caused by NameResolutionError(\\\"HTTPSConnection(host='login.microsoftonline.com', port=443): Failed to resolve 'login.microsoftonline.com' ([Errno -2] Name or service not known)\\\"))\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6667/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092109160337379328","occurred_at":"2026-08-25T04:37:46Z","signature_id":"bfc9e6d451ec941871c7fd17bc88e4377f14864e420626a1eb125b993b91f4d2","pr_number":6667,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc","failure_pattern":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","search_query":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-08-25T10:31:54.607Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description=\"Step swift-vnet\\n Kind: Shell\\n Command: ./swift-vnet.sh\\n\"\ntime=2026-08-25T10:31:55.356Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: Failed to load or parse file /tmp/azure-cli-config-3697365572/azureProfile.json. It will be overridden by default settings.\\nERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192723133009920","occurred_at":"2026-08-25T10:03:11Z","signature_id":"561505d61b3a52ebd516a8399919b0bdc92e7d97b736ef08c12b540f9f4f583c","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e43918bf51405cc433b6fba69bdc1ea9b482230892e03b1d2b08eabffb49ed22","failure_pattern":"VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: failed to verify image registry is disabled on cluster disabled-image-registry-hcp-cluster\nUnexpected error:\n \u003c*errors.joinError | 0xc0012d80d8\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n VerifyBasicAccess failed: failed to list services: Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: failed to verify image registry is disabled on cluster disabled-image-registry-hcp-cluster\nUnexpected error:\n \u003c*errors.joinError | 0xc0012d80d8\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n VerifyBasicAccess failed: failed to list services: Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","signature_id":"4654f560dff0807befbb619dbe35d2726f4f20abdaaa079b608640db38766779","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":16,"also_in":["STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":74,"prior_last_seen_at":"2026-08-21T21:06:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:220]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000012048\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22, targetMinor=5.0) failed: clusterversion status.history has no version in target minor \"5.0\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:220]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000012048\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22, targetMinor=5.0) failed: clusterversion status.history has no version in target minor \"5.0\"\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"9022b451906b9a6a0432bfd8f6a4bba8e8dea2ae8057ca72799a51fd124f9bca","pr_number":6679,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"70db2d61c2cd057fcd1ea1e913f1f94a57b2c2f5dbd99a0ba89a7c674d643f54","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j8947456 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j8947456 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j8947456 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j8947456 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:54:58Z\n Ended: 2026-08-25T16:00:59Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"c8a66989bf0343b565fa3da314117aa183d7f0a34150a848cecfbcf5f963353a","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0e72a5ca42c3a2d32bb83d02da4302579e640eee948d64f58a4f543c892be49b","failure_pattern":"alert [infra] Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j7405952 fired","search_query":"alert [infra] Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j7405952 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j7405952 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j7405952 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:54:11Z\n Ended: 2026-08-25T16:01:15Z\n Severity: Sev3\n Description: Cosmos DB returned more than 100 throttled requests (HTTP 429 - request rate too large) summed over a 5-minute window, evaluated every minute. Sustained 429s mean the workload is exceeding provisioned throughput - investigate hot partitions or increase provisioned throughput (RU/s). A low, steady rate of 429s that client SDKs retry transparently can be normal. https://learn.microsoft.com/azure/cosmos-db/troubleshoot-request-rate-too-large"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261270257405952","occurred_at":"2026-08-25T14:42:10Z","signature_id":"0766b6645c6206a3fc1a103b5ed7d9fe07fd1625b37249763edfe4af9b817fbb","pr_number":6614,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":51,"prior_last_seen_at":"2026-08-21T05:44:12Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T14:40:10Z\n Ended: 2026-08-24T14:49:11Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6438784-svc\", component=\"backend\", name=\"csstatedump\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879927736438784","occurred_at":"2026-08-24T13:26:52Z","signature_id":"f42ba6cef1b20012612134284fc91db6cdee01f2a97291f342457df1df0028e6","pr_number":6433,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-18T19:17:13Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-24T16:04:45Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n Description: Container fluentbit in pod arobit/arobit-forwarder-cxwt8 on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-08-24T16:04:46Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-jbtcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1ec91183-7c07-473b-b1d1-9df325a61c81\"\n Description: Container fluentbit in pod arobit/arobit-forwarder-jbtcb on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-08-24T16:04:46Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-jbtcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1ec91183-7c07-473b-b1d1-9df325a61c81\"\n Description: Container fluentbit in pod arobit/arobit-forwarder-jbtcb on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-08-24T16:04:46Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n Description: Container fluentbit in pod arobit/arobit-forwarder-cxwt8 on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"c239e70f5bf7ad0e42856cb2b9acc6d7c10a9ba7d6c9a4f5496cbd3609446018","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a89f0d2edcd262580a6fd914503ced16ad52ae763f2c2255b4f5af55d375ecd3","failure_pattern":"alert [svc] KubeContainerWaiting fired","search_query":"alert [svc] KubeContainerWaiting fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-21T03:34:17Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","occurrences":1}],"full_error_samples":["alert fired 8 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"liveness-probe\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-2k5cw\", severity=\"warning\"\n Description: pod/aks-secrets-store-csi-driver-2k5cw in namespace kube-system on container liveness-probe has been in waiting state for longer than 1 hour.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"trivy-scanner\", namespace=\"kube-system\", pod=\"eraser-aks-system-30244119-vmss000000-7fwv2\", severity=\"warning\"\n Description: pod/eraser-aks-system-30244119-vmss000000-7fwv2 in namespace kube-system on container trivy-scanner has been in waiting state for longer than 1 hour.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", namespace=\"kube-system\", pod=\"ama-metrics-node-rvqrk\", severity=\"warning\"\n Description: pod/ama-metrics-node-rvqrk in namespace kube-system on container prometheus-collector has been in waiting state for longer than 1 hour.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"secrets-store\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-2k5cw\", severity=\"warning\"\n Description: pod/aks-secrets-store-csi-driver-2k5cw in namespace kube-system on container secrets-store has been in waiting state for longer than 1 hour.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"addon-token-adapter\", namespace=\"kube-system\", pod=\"ama-metrics-node-rvqrk\", severity=\"warning\"\n Description: pod/ama-metrics-node-rvqrk in namespace kube-system on container addon-token-adapter has been in waiting state for longer than 1 hour.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"collector\", namespace=\"kube-system\", pod=\"eraser-aks-system-30244119-vmss000000-7fwv2\", severity=\"warning\"\n Description: pod/eraser-aks-system-30244119-vmss000000-7fwv2 in namespace kube-system on container collector has been in waiting state for longer than 1 hour.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"remover\", namespace=\"kube-system\", pod=\"eraser-aks-system-30244119-vmss000000-7fwv2\", severity=\"warning\"\n Description: pod/eraser-aks-system-30244119-vmss000000-7fwv2 in namespace kube-system on container remover has been in waiting state for longer than 1 hour.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"node-driver-registrar\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-2k5cw\", severity=\"warning\"\n Description: pod/aks-secrets-store-csi-driver-2k5cw in namespace kube-system on container node-driver-registrar has been in waiting state for longer than 1 hour."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092198034401136640","occurred_at":"2026-08-25T10:30:42Z","signature_id":"41f7f74865411c0e7be6f2be5d79bdefecee7e90cea861519501a2ae2711a449","pr_number":6433,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","failure_pattern":"alert [svc] KubeDaemonSetMisScheduled fired","search_query":"alert [svc] KubeDaemonSetMisScheduled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":65,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":207,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","occurrences":1}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 3:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:23:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 4:\n State: Resolved\n Started: 2026-08-24T16:13:11Z\n Ended: 2026-08-24T16:23:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"7caa0010541a7c4ff937ecc7f1700635aa06e8c5ef418d2f4d2c09799f427a78","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":28,"prior_last_seen_at":"2026-08-22T04:44:09Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 6 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T18:15:41Z\n Ended: 2026-08-24T18:21:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T18:15:41Z\n Ended: 2026-08-24T18:22:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-24T18:15:41Z\n Ended: 2026-08-24T18:22:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-24T18:15:41Z\n Ended: 2026-08-24T18:23:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-24T18:15:41Z\n Ended: 2026-08-24T18:22:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-24T18:15:41Z\n Ended: 2026-08-24T18:23:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"a6a83541f08b4359551ade88cceb74431bde5d979f13cb6852b8c8bc1013fd98","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d6c6b690ca60846dd98e464313d6a9672b9abd8744604edd915627eb68203f9d","failure_pattern":"alert [svc] KubeNodeNotReady fired","search_query":"alert [svc] KubeNodeNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":52,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":183,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T11:52:32Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeNotReady\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"prometheus\", node=\"aks-system-30244119-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n Description: aks-system-30244119-vmss000000 has been unready for more than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-08-25T11:52:32Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeNotReady\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"prometheus\", node=\"aks-system-30244119-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n Description: aks-system-30244119-vmss000000 has been unready for more than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092198034401136640","occurred_at":"2026-08-25T10:30:42Z","signature_id":"7ebd8243470fe8b709d9e6b75f707a2eff5fb9d0f87bf99b81e7e336da9429c9","pr_number":6433,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","failure_pattern":"alert [svc] KubeNodeUnreachable fired","search_query":"alert [svc] KubeNodeUnreachable fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":53,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":172,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T16:12:59Z\n Ended: 2026-08-24T16:22:56Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"prometheus\", node=\"aks-userswft2-20578738-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: aks-userswft2-20578738-vmss000002 is unreachable and some workloads may be rescheduled.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T16:12:59Z\n Ended: 2026-08-24T16:23:55Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"prometheus\", node=\"aks-userswft2-20578738-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: aks-userswft2-20578738-vmss000002 is unreachable and some workloads may be rescheduled."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"8ea4612bb5240c0f449b27c94dc170f61dc652ef24bc74880e6489a1d15ea2be","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-13T06:28:29Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T16:19:07Z\n Ended: 2026-08-24T16:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n Description: Pod arobit/arobit-forwarder-cxwt8 (fluentbit) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-08-24T16:20:06Z\n Ended: 2026-08-24T16:33:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n Description: Pod arobit/arobit-forwarder-cxwt8 (fluentbit) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"fe91f4f69f02671271e4cc6e4aaa4f966388d16d9e660f0bf8e6f1f0d5179d16","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-20T23:44:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092147015625478144","occurred_at":"2026-08-25T07:08:13Z","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","pr_number":6669,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5dd5ef6724beffab1f42a094be91cb72cdbac22ffe68c01aeeec9da825c19fa3","failure_pattern":"error for empty VM name should contain 400 status code","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:359]: error for empty VM name should contain 400 status code","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:359]: error for empty VM name should contain 400 status code\nExpected\n \u003cstring\u003e: ...\nto contain substring\n \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:359]: error for empty VM name should contain 400 status code\nExpected\n \u003cstring\u003e: ...\nto contain substring\n \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"2f2e10c3f96af5d17d27eaefc2cdd8f45ad43cf97747092ff970d29300a33cb8","pr_number":6660,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"95452bdc0ded378689e5f2c12b2d45cbf916f7a4e0676d618a92d0f3d8778ac4","failure_pattern":"failed to read rendered config /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: open /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: no such file or directory","search_query":"failed to read rendered config /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: open /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: no such file or directory","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: failed to load rendered config — is RENDERED_CONFIG set?\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eafac0\u003e: \n failed to read rendered config /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: open /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: no such file or directory\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: failed to load rendered config — is RENDERED_CONFIG set?\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eafac0\u003e: \n failed to read rendered config /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: open /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: no such file or directory\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"18cefbe16bc079f2ead9cb47f111db79d94e08b882e7b91731e46458212a977b","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":15,"prior_last_seen_at":"2026-08-19T16:39:13Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-08-24T09:44:43.919Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T09:44:44.928Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-08-24T09:44:46.104Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=f40d7b7ab0b2d9646dc8b78fbaa252859442c0d4d4549c8765d66346f06ac123 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4585344-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff40d7b7ab0b2d9646dc8b78fbaa252859442c0d4d4549c8765d66346f06ac123\ntime=2026-08-24T10:14:43.424Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6655/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091810349014585344","occurred_at":"2026-08-24T08:50:25Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6655,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"911f609de0f6334698a9ae03ee21b22f86f10fb03ad2176871cac7bbc4b43a5f","failure_pattern":"failed to select node pool \u003cnodepool\u003e version for \u003cocp-channel\u003e channel: Get \"\u003curl\u003e\": dial tcp: lookup api","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-20T16:56:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.208.156:42093-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.208.156:42093-\u003e172.30.0.10:53: i/o timeout"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"7171e132341694fb10089af1503bd1696a386e558f7ad492e46e056500959d3b","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":14,"prior_last_seen_at":"2026-08-21T21:06:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:186]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:186]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071623086575616","occurred_at":"2026-08-25T02:08:38Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6658,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5d433008aba23c344781b86d474b8dbf122c3c1ce0be7c5095e73ef5b595140f","failure_pattern":"run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"\u003cvm\u003e\" (output: \"Client Version: v1.36.4 Kustomize Version: v5.8.1\")","search_query":"run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:160]: kubectl version should succeed from VM via private KAS internal LB (output: )\nUnexpected error:\n \u003c*errors.errorString | 0xc0007fbf60\u003e: \n run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:160]: kubectl version should succeed from VM via private KAS internal LB (output: )\nUnexpected error:\n \u003c*errors.errorString | 0xc0007fbf60\u003e: \n run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879954055696384","occurred_at":"2026-08-24T13:27:00Z","signature_id":"d079461036286dee8f18df87fbaabb5b38f1fcaa64aea1bbd3a2fe3d558a9ddb","pr_number":6429,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bff39f67862ef72eccdd3c2006d48756f417856635339107ace9fde9b0a679a4","failure_pattern":"stamp 2 scheduling must have requested resources","search_query":"stamp 2 scheduling must have requested resources","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have scheduling data for ready management clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.4166666666666667,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have scheduling data for ready management clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:155 with:\nstamp 2 scheduling must have requested resources\nExpected\n \u003cv1.ResourceList | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:155 with:\nstamp 2 scheduling must have requested resources\nExpected\n \u003cv1.ResourceList | len:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"b261dd52f676523d8a989fb0761bdfb6871af15b6d8ed758e7d327e30440bdf8","pr_number":6657,"after_last_push_of_merged_pr":false}]}]},{"environment":"int","summary":{"total_runs":7,"failed_runs":3,"raw_occurrences":4,"matched_occurrences":3,"runs_affected":2},"rows":[{"environment":"int","failure_pattern_id":"13ca13618a2c4b8b7429b3001f7d02e419916c0961928bfe37af66198711655d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":7,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-03"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-07T23:58:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e43900\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fbxfthcgbs8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n \"name\": \"7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T10:43:39.73808157Z\",\n \"endTime\": \"2026-08-24T11:02:45.476245961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e43900\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fbxfthcgbs8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n \"name\": \"7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T10:43:39.73808157Z\",\n \"endTime\": \"2026-08-24T11:02:45.476245961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091834116805758976","occurred_at":"2026-08-24T10:24:52Z","signature_id":"3fa15812aed9d226c2b6bdead73b3a2c80d16497f78ee163ab8cbfa1074904ed","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"d414eaa781a1a73b840b0ceb0c494d2cfb36d6ce2f01987a892eec84026f3876","failure_pattern":"failed to select node pool \u003cnodepool\u003e version for \u003cocp-channel\u003e channel: Get \"\u003curl\u003e\": net/http: TLS handsh","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handsh","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handshake timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handshake timeout"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091834116805758976","occurred_at":"2026-08-24T10:24:52Z","signature_id":"c4114fce72966a1339b4e9230a4f088e9e9d8e5a8939f917174c276b7d0a2542","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":7,"also_in":["STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-20T16:42:49Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-l8xph5\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f20140\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-trf9f.apps.aro.r8g4l2d8f3g0v2i.b5du.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-l8xph5\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f20140\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-trf9f.apps.aro.r8g4l2d8f3g0v2i.b5du.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091735435544891392","occurred_at":"2026-08-24T03:52:45Z","signature_id":"571e7c77690730a14fa42704d42877387a3fe5f21673f5f238b11729008a3c86","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":11,"failed_runs":6,"raw_occurrences":21,"matched_occurrences":20,"runs_affected":5},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-gm6hwh, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: failed waiting for cluster=\"cluster-candidate-5-0-gm6hwh\" in resourcegroup=\"rg-candidate-5-0-mqrq7j46wgxv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":27.272727272727273,"anchor_occurrences":4,"after_last_push_count":4,"also_in":["STG"],"trend_counts":[0,3,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":33,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-zmwf7l76v4hv/cluster-candidate-5-0-p2dtvg should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000a42420\u003e: \n failed to create HCP cluster cluster-candidate-5-0-p2dtvg, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: failed waiting for cluster=\"cluster-candidate-5-0-p2dtvg\" in resourcegroup=\"rg-candidate-5-0-zmwf7l76v4hv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-zmwf7l76v4hv/cluster-candidate-5-0-p2dtvg should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000a42420\u003e: \n failed to create HCP cluster cluster-candidate-5-0-p2dtvg, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: failed waiting for cluster=\"cluster-candidate-5-0-p2dtvg\" in resourcegroup=\"rg-candidate-5-0-zmwf7l76v4hv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001f06450\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-m4qs88j7zlw8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001f06450\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-m4qs88j7zlw8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-c58j4775b54x/cluster-nightly-5-1-rb2jwr should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001a0d800\u003e: \n failed to create HCP cluster cluster-nightly-5-1-rb2jwr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: failed waiting for cluster=\"cluster-nightly-5-1-rb2jwr\" in resourcegroup=\"rg-nightly-5-1-c58j4775b54x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-c58j4775b54x/cluster-nightly-5-1-rb2jwr should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001a0d800\u003e: \n failed to create HCP cluster cluster-nightly-5-1-rb2jwr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: failed waiting for cluster=\"cluster-nightly-5-1-rb2jwr\" in resourcegroup=\"rg-nightly-5-1-c58j4775b54x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","signature_id":"f812b5873fe0043e9314ca322d6d4251f24251a8ffe19fbea9df860ac64ea470","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"48cf5ec4badd8d65b424a299cc4f30c373af95f5d42c09e6725e0ee15b8c1667","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"a75e2b44cb494e9ca286b15ddee1a744c32c76fd89cb2c425050bd89a5961c7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"5fc7819289b1249d9f2299f61167ea9b6a16c06d2102e06da9f3562ee86b0ad0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"97958430e3a844b6f5be05213043088dddda4827bc264aa7b71a4bdf2cce51d5","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":18.181818181818183,"anchor_occurrences":3,"after_last_push_count":10,"also_in":["DEV"],"trend_counts":[1,0,3,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00093afe0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-bm5z6ztxsxcf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56042e31-0efc-4287-925a-c64efa21388d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56042e31-0efc-4287-925a-c64efa21388d\",\n \"name\": \"56042e31-0efc-4287-925a-c64efa21388d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:30:08.889821647Z\",\n \"endTime\": \"2026-08-25T10:49:13.717723647Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00093afe0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-bm5z6ztxsxcf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56042e31-0efc-4287-925a-c64efa21388d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56042e31-0efc-4287-925a-c64efa21388d\",\n \"name\": \"56042e31-0efc-4287-925a-c64efa21388d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:30:08.889821647Z\",\n \"endTime\": \"2026-08-25T10:49:13.717723647Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006d1700\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dnqrhz4h5n9n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26ec1bc7-0ddd-42ef-916c-12f4d896455c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26ec1bc7-0ddd-42ef-916c-12f4d896455c\",\n \"name\": \"26ec1bc7-0ddd-42ef-916c-12f4d896455c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:34:58.109914662Z\",\n \"endTime\": \"2026-08-25T07:53:59.934466955Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006d1700\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dnqrhz4h5n9n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26ec1bc7-0ddd-42ef-916c-12f4d896455c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26ec1bc7-0ddd-42ef-916c-12f4d896455c\",\n \"name\": \"26ec1bc7-0ddd-42ef-916c-12f4d896455c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:34:58.109914662Z\",\n \"endTime\": \"2026-08-25T07:53:59.934466955Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a308e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ks6pxxnlhsjm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc3b6654-559a-489d-a192-908217a06b8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc3b6654-559a-489d-a192-908217a06b8f\",\n \"name\": \"fc3b6654-559a-489d-a192-908217a06b8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:33:44.141359446Z\",\n \"endTime\": \"2026-08-25T07:52:49.875150436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a308e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ks6pxxnlhsjm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc3b6654-559a-489d-a192-908217a06b8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc3b6654-559a-489d-a192-908217a06b8f\",\n \"name\": \"fc3b6654-559a-489d-a192-908217a06b8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:33:44.141359446Z\",\n \"endTime\": \"2026-08-25T07:52:49.875150436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092192947897372672","occurred_at":"2026-08-25T10:10:43Z","signature_id":"ad66320c5357dd06c4fac6f8b26348313ed54ba4b4268b4e00a204574fabedc1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092146624879923200","occurred_at":"2026-08-25T07:06:40Z","signature_id":"16e9f20862be2ffa8d9254ea133442622a04c2230e897bc0c633345d949e6e0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092146624879923200","occurred_at":"2026-08-25T07:06:40Z","signature_id":"837b614da929eed13cefb96defae6f8e2c3ed1ab2105b3725cca3c17f8b63fa0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"beaf47bd37dc21f47808a0240288ea4322da110d92ca67bf08de0f6793353753","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.35; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":18.181818181818183,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[1,1,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-2m6hjk\" to minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00144a750\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6dbxgwtzxfsh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-2m6hjk\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-2m6hjk\" to minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00144a750\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6dbxgwtzxfsh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-2m6hjk\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-96fcvx\" to minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0008e5080\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-qkzx8x2fwwlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-96fcvx\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-96fcvx\" to minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0008e5080\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-qkzx8x2fwwlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-96fcvx\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","signature_id":"6b7decf97f628a3a86f9cc81aac7b2b793ce71dbec005f9c0bbc92ad65ed2200","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"67be8c6caed95a193a1745532d025891093937507a496524f899a7d2c80f89f2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5443585aedc88f5d2589a8fb3795e77266206bcfdb680d3cfbedccaf7bc67f50","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.30; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":18.181818181818183,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[1,1,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-hc729b\" to minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000cc7e30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4j8dsngwmfpg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-hc729b\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-hc729b\" to minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000cc7e30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4j8dsngwmfpg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-hc729b\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-znvqfb\" to minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0008e8630\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-bb24w6nmgn9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-znvqfb\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-znvqfb\" to minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0008e8630\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-bb24w6nmgn9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-znvqfb\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","signature_id":"47e6560075ab2811dc4afc63d80fcd866f7a8dbf54b0f240501f8704720bfe02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"eabd42ab0d34b7f1d8ca082eb227c6c5a44990af4cb2ad49ebe668cc8663e6ed","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"2b6de8e3eeb7b1eb9cb9733a09ec9f907952649e2f47b910e6bb29761a829864","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.10; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":18.181818181818183,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[1,1,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-qp64bq\" to minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a2bec0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-dqpzmwzqshp6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-qp64bq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-qp64bq\" to minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a2bec0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-dqpzmwzqshp6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-qp64bq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-gmlx25\" to minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000703a40\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b6dm7hck8ksd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-gmlx25\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-gmlx25\" to minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000703a40\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b6dm7hck8ksd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-gmlx25\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","signature_id":"5f57af3b362f707a2fd214f4c7abae7f575a8b66d80b93974a89ebabf4108803","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"233817672ec9f15c61036dd69be2232e5da454215c4ae03262d5d9d1147fe964","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6c0df54cac07833305ae54d7a30bcf9a55390b618040466bd5f2843d80e698a8","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":18.181818181818183,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[1,1,1,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000b1c2d0\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000b1c2d0\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000ad6720\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000ad6720\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","signature_id":"328ab83e407af59a94692dc615a105004e707662f379710e72fd8156cdefd247","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"3fd31e6b902a75b339639f3cb28f3e152b295a16939ec2f7056ea2da7a8f4e73","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f966d8a6f1dc892e4c6ee21571e53ba7d8d1ed39b375430e469f3616f54b9b95","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-7z565t\" to 4.21.0-0.nightly-multi-2026-08-24-143120\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b5de0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-7z565t\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-98477z6czjf7\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n \"name\": \"d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:45:23.595556181Z\",\n \"endTime\": \"2026-08-24T23:45:30.682388649Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-7z565t\" to 4.21.0-0.nightly-multi-2026-08-24-143120\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b5de0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-7z565t\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-98477z6czjf7\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n \"name\": \"d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:45:23.595556181Z\",\n \"endTime\": \"2026-08-24T23:45:30.682388649Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"15bac440b3d4b1dd68415a557e13f0d806f6cdb816f9edcd9423e02924b01f2e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d8c04d75e2d056f1b8b36668a13ab7045b056c9f740c7013c4882cfcfef5d2d6","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-cbj2dm\" to 4.22.0-0.nightly-multi-2026-08-23-080550\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df2720\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-cbj2dm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4ls4ksxqchgg\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n \"name\": \"d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:48:14.416516082Z\",\n \"endTime\": \"2026-08-24T23:48:20.787265431Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-cbj2dm\" to 4.22.0-0.nightly-multi-2026-08-23-080550\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df2720\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-cbj2dm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4ls4ksxqchgg\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n \"name\": \"d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:48:14.416516082Z\",\n \"endTime\": \"2026-08-24T23:48:20.787265431Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"498a13bc157b497dedc0c407f7c032fb09b28d4110c6fbee95fdae33ff5fe349","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"724e7a51f2cd05d0f0c351e24f28344ba4eb40490e6de7ed02ace0e7a7856472","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[1,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-lrdz8v\" to 4.23.0-0.nightly-multi-2026-08-24-110923\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000881340\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-lrdz8v\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-t25q8kvqvdhw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\",\n \"name\": \"4393d587-2d85-4674-a4f6-155ceb1138f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:48:28.850341951Z\",\n \"endTime\": \"2026-08-24T23:48:30.944709051Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-lrdz8v\" to 4.23.0-0.nightly-multi-2026-08-24-110923\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000881340\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-lrdz8v\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-t25q8kvqvdhw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\",\n \"name\": \"4393d587-2d85-4674-a4f6-155ceb1138f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:48:28.850341951Z\",\n \"endTime\": \"2026-08-24T23:48:30.944709051Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"6663bc04df6e96cb243dd9f35fd825b118edab670cd360f1ba5fc79ade7a0f44","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3b509588f678a0fd976eb453004a3e8a55c0be5c12f8340c463453a657ce6678","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-q6hwb9\" to 5.1.0-0.nightly-multi-2026-08-24-111752\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a25da0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-q6hwb9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-kgxwzfcjcc2b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n \"name\": \"7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:51:24.1449981Z\",\n \"endTime\": \"2026-08-24T23:51:30.86207539Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-q6hwb9\" to 5.1.0-0.nightly-multi-2026-08-24-111752\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a25da0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-q6hwb9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-kgxwzfcjcc2b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n \"name\": \"7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:51:24.1449981Z\",\n \"endTime\": \"2026-08-24T23:51:30.86207539Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"05526aa86d5e4dd680e08fb3cf223532281a30bb281575b5c8abfec129525d60","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6411b2b281a97b7e65400b269143638faa6befdec2dd319ac1327622333be049","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":5,"also_in":["STG"],"trend_counts":[1,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002b4c960\u003e: \n node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1.35.4\"); o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002b4c960\u003e: \n node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1.35.4\"); o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"0c1e40a7688ce333e1802f99689a7431268dce71db699919518e6ab02caaba1e","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":6,"failed_runs":3,"raw_occurrences":20,"matched_occurrences":19,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":8,"after_last_push_count":55,"also_in":["INT"],"trend_counts":[18,8,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001990018\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-58vq7.apps.aro.cilium-cluster.xjbu.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001990018\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-58vq7.apps.aro.cilium-cluster.xjbu.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc0023089a8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gd2m6.apps.aro.agg-cluster.sh9x.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc0023089a8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gd2m6.apps.aro.agg-cluster.sh9x.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc002bc60a8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2xkbq.apps.aro.cilium-cl.yqg3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc002bc60a8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2xkbq.apps.aro.cilium-cl.yqg3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"128bdc5bd9fd4d676037c7b3876358423100506abaac1cb7c6a2aa6dca99f9d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"268af4cfefc9d9ce5477c6309e34d9e625934c5fec5612737f144e143459ca23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"568397239d373e84cc1193077022f2929fdbc7536d41e0a023cda085a91923ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"64d7370897c1ddb0a4c082a98b5de0abd894ee5bd836383e651a1f53f91a033d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"6f400eea163c9e3d28c8b1f64accc1cf5963abf78c260413f5fe388a60f0006c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"aa4dc686c4f2ffb192f550321a5d618c671a84350ba22f26b4ee1aaac5686fb0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"c217516f086c4ee518f66b5fc0b63d7e19723573004fb051ee9ac9e2e83e0fb9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"de74f8de034a6e8da10a240f9e0d6f03ee3be2522666aaf0b6ad709ddf3efbc4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":4,"after_last_push_count":19,"also_in":["DEV"],"trend_counts":[4,4,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0012c61b0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-23-080550, targetMinor=4.23.0-0.nightly-multi-2026-08-24-110923) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-24-110923\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0012c61b0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-23-080550, targetMinor=4.23.0-0.nightly-multi-2026-08-24-110923) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-24-110923\"\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000013c68\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-24-135049, targetMinor=4.21.0-0.nightly-multi-2026-08-24-143120) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-24-143120\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000013c68\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-24-135049, targetMinor=4.21.0-0.nightly-multi-2026-08-24-143120) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-24-143120\"\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001423c38\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-24-143120, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001423c38\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-24-143120, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"1f961d63e5efbc9f674297ab6ff360e18191616cad68297a32bfc0352b1efd45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"5d51d6400541f1ef2ea735bc34f88c2cca7f1b91d8f8680efed7dfe0ab6e075d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"8c228d79217e5703c43c091d1bbcb6c74c0d2e48dd70a8ed3bbc113b8600d236","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"9a6af90baac0bf0367bb530e542307559248b48af09dc4fa5550242ec09f64e4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":146,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":13,"prior_last_seen_at":"2026-08-21T23:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8ea40\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf6jg45wtwxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\",\n \"name\": \"62db7451-86ec-4c6a-9227-bc775f891ebb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:05:01.131009318Z\",\n \"endTime\": \"2026-08-24T23:24:05.637766062Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8ea40\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf6jg45wtwxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\",\n \"name\": \"62db7451-86ec-4c6a-9227-bc775f891ebb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:05:01.131009318Z\",\n \"endTime\": \"2026-08-24T23:24:05.637766062Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"ce363ec1f1315846ac4ca18a663f8ca57451d8f5854815877a996fbdf3f45dd9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"db86d0a7b49ef93daf7cdb63d12e83b79ddcd71210e2847a436b8d80dd5dde7f","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] \u003cno_message\u003e; [clusterServiceStatus] \u003cno_message\u003e; [descendantResources] \u003cno_message\u003e; [hostedCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001716270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nml9dcvzzfr9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n \"name\": \"d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:14:11.789973414Z\",\n \"endTime\": \"2026-08-24T23:38:15.678890881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001716270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nml9dcvzzfr9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n \"name\": \"d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:14:11.789973414Z\",\n \"endTime\": \"2026-08-24T23:38:15.678890881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"3feed0e49b703d23a4823f602dec7fe62c46bf78411b06f90b5ee2c2bde72da5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-20T08:35:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc00118e480\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-h759ch\" in resourcegroup=\"rg-nightly-4-20-mzr4r5lz42qp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\",\n \"name\": \"55ed743f-d106-4adb-b590-caa3369caeb3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:25:48.884097042Z\",\n \"endTime\": \"2026-08-24T23:49:50.709200875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd7vm7aq4os1ksjghr3o1msj3ksno6l still exists (deletion dispatched at 2026-08-24T23:25:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc00118e480\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-h759ch\" in resourcegroup=\"rg-nightly-4-20-mzr4r5lz42qp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\",\n \"name\": \"55ed743f-d106-4adb-b590-caa3369caeb3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:25:48.884097042Z\",\n \"endTime\": \"2026-08-24T23:49:50.709200875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd7vm7aq4os1ksjghr3o1msj3ksno6l still exists (deletion dispatched at 2026-08-24T23:25:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"6a0460b71f8ba99286143da10d9b371325b39609e9612fa043b5980a861407e3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b6258cd488b080bf5ba7099ce8387b9d2f4d222d926f9a0c81022dc736e89db2","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[2,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0017be510\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0017be510\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"c14f9a5639d319819baa7a308cbb789519348e32b0abac6b02f5611b9187fbe7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c75b40567502299dd397c0248cce6097e2377df763d3deec7ae8c7f9cd09e3b2","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0012c0d10\u003e: \n node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]},{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0012c0d10\u003e: \n node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]},{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"5565031e089c6eb68fe43eb98d2d43776a82305498e8c6a0a7f4c918b0348bf0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":26,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002ac4480\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002ac4480\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"1ecfdcaa8fd2583c7f65498931069ae0e42fa1b933a7ed7714a974ea82a7b373","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[2,1,0,0,0,0,0,0],"trend_range":"2026-08-23..2026-08-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003cx509.UnknownAuthorityError\u003e: \n x509: certificate signed by unknown authority\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003cx509.UnknownAuthorityError\u003e: \n x509: certificate signed by unknown authority\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}