{"meta":{"start_date":"2026-06-11","end_date":"2026-06-11","start_at":"2026-06-11T00:00:00Z","end_at":"2026-06-12T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T11:01:42Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":11,"failed_runs":9,"raw_occurrences":16,"matched_occurrences":16,"runs_affected":9},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-nightly-4-23-pb9qnq\" in resourcegroup=\"rg-nightly-4-23-9kfvlt\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-pb9qnq in resource group rg-nightly-4-23-9kfvlt, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":27.272727272727273,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-06-05..2026-06-11 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25","2026-06-01"],"prior_runs_affected":32,"prior_last_seen_at":"2026-06-07T23:00:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-v9zfnv/cluster-nightly-5-0-jq7d6s should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001510ae0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-jq7d6s\" in resourcegroup=\"rg-nightly-5-0-v9zfnv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-jq7d6s in resource group rg-nightly-5-0-v9zfnv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-v9zfnv/cluster-nightly-5-0-jq7d6s should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001510ae0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-jq7d6s\" in resourcegroup=\"rg-nightly-5-0-v9zfnv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-jq7d6s in resource group rg-nightly-5-0-v9zfnv, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-krdd6q/cluster-nightly-4-23-q28cpq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0018073b0\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-q28cpq\" in resourcegroup=\"rg-nightly-4-23-krdd6q\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-q28cpq in resource group rg-nightly-4-23-krdd6q, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-krdd6q/cluster-nightly-4-23-q28cpq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0018073b0\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-q28cpq\" in resourcegroup=\"rg-nightly-4-23-krdd6q\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-q28cpq in resource group rg-nightly-4-23-krdd6q, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-7mghfc/cluster-nightly-4-23-qxb2mc should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014f5020\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-qxb2mc\" in resourcegroup=\"rg-nightly-4-23-7mghfc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-qxb2mc in resource group rg-nightly-4-23-7mghfc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-7mghfc/cluster-nightly-4-23-qxb2mc should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014f5020\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-qxb2mc\" in resourcegroup=\"rg-nightly-4-23-7mghfc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-qxb2mc in resource group rg-nightly-4-23-7mghfc, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065207457541525504","occurred_at":"2026-06-11T23:00:02Z","signature_id":"8c84792dc7fc683c4bc2e2b7121cd7aa8a85319227bab7844b1708bff445d875","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065207457541525504","occurred_at":"2026-06-11T23:00:02Z","signature_id":"cc537cd5eaacd194e2cfb4e930b3faa948a61ddfe8751351edbb2ff400fcb2c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065133461022183424","occurred_at":"2026-06-11T18:06:00Z","signature_id":"897ea65756daec9faa905677c938f13528c412af6902d75e6affb822376462e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065133461022183424","occurred_at":"2026-06-11T18:06:00Z","signature_id":"af1dfca2123bd3256e8df77d8db24de16ca4e7421a2d964520820b0febd56690","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064999166156738560","occurred_at":"2026-06-11T09:12:21Z","signature_id":"1f562dd8dfe058b5cb72dc24c089d783b528b54c85a0581c4dc141522b3d8d07","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":36.36363636363637,"anchor_occurrences":4,"after_last_push_count":17,"trend_counts":[0,0,0,2,3,2,4],"trend_range":"2026-06-05..2026-06-11 UTC","prior_windows_present":2,"prior_window_starts":["2026-05-18","2026-05-25"],"prior_runs_affected":13,"prior_last_seen_at":"2026-05-30T02:00:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:37]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:37]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064995434417360896","occurred_at":"2026-06-11T08:57:31Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064979974896488448","occurred_at":"2026-06-11T07:56:06Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064892701127479296","occurred_at":"2026-06-11T02:09:18Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064892559024459776","occurred_at":"2026-06-11T02:08:44Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-2cx2hr in resource group rg-candidate-4-20-zgqpq8, error: failed waiting for nodepool=\"np-2cx2hr\" for cluster \"cluster-candidate-4-20-2cx2hr\" in resourcegroup=\"rg-candidate-4-20-zgqpq8\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":27.272727272727273,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-06-05..2026-06-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25"],"prior_runs_affected":10,"prior_last_seen_at":"2026-05-27T23:14:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-qdg249\" for cluster \"cluster-candidate-4-21-qdg249\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009ad380\u003e: \n    failed to create NodePool np-qdg249, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-qdg249 in resource group rg-candidate-4-21-vrs8xc, error: failed waiting for nodepool=\"np-qdg249\" for cluster \"cluster-candidate-4-21-qdg249\" in resourcegroup=\"rg-candidate-4-21-vrs8xc\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-qdg249\" for cluster \"cluster-candidate-4-21-qdg249\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009ad380\u003e: \n    failed to create NodePool np-qdg249, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-qdg249 in resource group rg-candidate-4-21-vrs8xc, error: failed waiting for nodepool=\"np-qdg249\" for cluster \"cluster-candidate-4-21-qdg249\" in resourcegroup=\"rg-candidate-4-21-vrs8xc\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-bbwmsl\" for cluster \"cluster-candidate-4-21-bbwmsl\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011f4360\u003e: \n    failed to create NodePool np-bbwmsl, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-bbwmsl in resource group rg-candidate-4-21-tf47dc, error: failed waiting for nodepool=\"np-bbwmsl\" for cluster \"cluster-candidate-4-21-bbwmsl\" in resourcegroup=\"rg-candidate-4-21-tf47dc\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-bbwmsl\" for cluster \"cluster-candidate-4-21-bbwmsl\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011f4360\u003e: \n    failed to create NodePool np-bbwmsl, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-bbwmsl in resource group rg-candidate-4-21-tf47dc, error: failed waiting for nodepool=\"np-bbwmsl\" for cluster \"cluster-candidate-4-21-bbwmsl\" in resourcegroup=\"rg-candidate-4-21-tf47dc\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-ntc7jm\" for cluster \"cluster-candidate-4-20-ntc7jm\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008abe00\u003e: \n    failed to create NodePool np-ntc7jm, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-ntc7jm in resource group rg-candidate-4-20-ggl9jk, error: failed waiting for nodepool=\"np-ntc7jm\" for cluster \"cluster-candidate-4-20-ntc7jm\" in resourcegroup=\"rg-candidate-4-20-ggl9jk\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-ntc7jm\" for cluster \"cluster-candidate-4-20-ntc7jm\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008abe00\u003e: \n    failed to create NodePool np-ntc7jm, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-ntc7jm in resource group rg-candidate-4-20-ggl9jk, error: failed waiting for nodepool=\"np-ntc7jm\" for cluster \"cluster-candidate-4-20-ntc7jm\" in resourcegroup=\"rg-candidate-4-20-ggl9jk\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064999166156738560","occurred_at":"2026-06-11T09:12:21Z","signature_id":"2941394c649e35c40080997ff35fd327f53cce95fd8fae48411c14dcd7b70e1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064892795172163584","occurred_at":"2026-06-11T02:09:40Z","signature_id":"35b0046c8024b6533ae4ef988d82fc15cc7483616dee9c1514a6e677dca73311","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064892795172163584","occurred_at":"2026-06-11T02:09:40Z","signature_id":"8eb6d39ad7f904f6f20fb2e538cbae5c0e47eff7450f825f0ebb13249cfbd826","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064892701127479296","occurred_at":"2026-06-11T02:09:18Z","signature_id":"85a48cb96dde09770771775b70187dd4765302c34f5c5c98a1cdd7d00c138e4b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"80f757ab340f6d75766a4d77caaaddd21d63ed517eb7806a91d6c90f11d6a624","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.19.33\": cannot downgrade from version 4.21.19 to 4.19.33; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-06-05..2026-06-11 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f04b20\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-c9l7d4\" resourcegroup=\"rg-np-dg-minor-c9l7d4-jlrlpp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-c9l7d4-jlrlpp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-c9l7d4/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.19 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f04b20\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-c9l7d4\" resourcegroup=\"rg-np-dg-minor-c9l7d4-jlrlpp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-c9l7d4-jlrlpp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-c9l7d4/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.19 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064890364723990528","occurred_at":"2026-06-11T02:00:01Z","signature_id":"d53636f590081bb00e9fd2ac616257940cd863f596f189021d78c5f0b277e8f4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b86f9a264db85a731f5c6fdee94b159cf165b041d6a81b701f673ae62312bfa8","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21.18\": cannot downgrade from version 4.21.19 to 4.21.18; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-06-05..2026-06-11 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.18\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00099a220\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-sh8sbm\" resourcegroup=\"rg-np-downgrade-sh8sbm-8xc2g7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-sh8sbm-8xc2g7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-sh8sbm/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": invalid node pool version 4.21.18: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": cannot downgrade from version 4.21.19 to 4.21.18\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.18\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00099a220\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-sh8sbm\" resourcegroup=\"rg-np-downgrade-sh8sbm-8xc2g7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-sh8sbm-8xc2g7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-sh8sbm/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": invalid node pool version 4.21.18: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": cannot downgrade from version 4.21.19 to 4.21.18\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064890364723990528","occurred_at":"2026-06-11T02:00:01Z","signature_id":"b13fe9b95b3c5336cfc484bd9bfc10633a6825001520493a6e3175e5cb7725f1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"baee198396fe46d09318662e813056f6a99076e40cebbc3f321bb246d7fd7f20","failure_pattern":"Post \"\u003curl\u003e\": dial tcp \u003cip\u003e:\u003cport\u003e: i/o timeout","search_query":"Post \"https://api.n1i3e7e9x8g0b6s.ci57.brazilsouth.aroapp-hcp.io:443/apis/authentication.k8s.io/v1/selfsubjectreviews\": dial tcp 74.163.139.192:443: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-05..2026-06-11 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:236]: should be able to create SelfSubjectReview for admin credential 2\nUnexpected error:\n    \u003c*url.Error | 0xc000999560\u003e: \n    Post \"https://api.n1i3e7e9x8g0b6s.ci57.brazilsouth.aroapp-hcp.io:443/apis/authentication.k8s.io/v1/selfsubjectreviews\": dial tcp 74.163.139.192:443: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:236]: should be able to create SelfSubjectReview for admin credential 2\nUnexpected error:\n    \u003c*url.Error | 0xc000999560\u003e: \n    Post \"https://api.n1i3e7e9x8g0b6s.ci57.brazilsouth.aroapp-hcp.io:443/apis/authentication.k8s.io/v1/selfsubjectreviews\": dial tcp 74.163.139.192:443: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064995434417360896","occurred_at":"2026-06-11T08:57:31Z","signature_id":"5c44ef45a71fd7d703be1cefc7a23cb14c5954fb9c9fedad6bb00fd770768efe","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}