{"meta":{"start_date":"2026-06-10","end_date":"2026-06-10","start_at":"2026-06-10T00:00:00Z","end_at":"2026-06-11T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T11:42:22Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":6,"failed_runs":6,"raw_occurrences":11,"matched_occurrences":11,"runs_affected":6},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-nightly-5-0-5r4lh5\" in resourcegroup=\"rg-nightly-5-0-kcn82j\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-5r4lh5 in resource group rg-nightly-5-0-kcn82j, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":50,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-06-04..2026-06-10 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25","2026-06-01"],"prior_runs_affected":29,"prior_last_seen_at":"2026-06-07T23:00:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-c7x88v/cluster-nightly-5-0-msgf8g should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00053a0f0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-msgf8g\" in resourcegroup=\"rg-nightly-5-0-c7x88v\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-msgf8g in resource group rg-nightly-5-0-c7x88v, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-c7x88v/cluster-nightly-5-0-msgf8g should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00053a0f0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-msgf8g\" in resourcegroup=\"rg-nightly-5-0-c7x88v\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-msgf8g in resource group rg-nightly-5-0-c7x88v, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-kcn82j/cluster-nightly-5-0-5r4lh5 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001322450\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-5r4lh5\" in resourcegroup=\"rg-nightly-5-0-kcn82j\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-5r4lh5 in resource group rg-nightly-5-0-kcn82j, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-kcn82j/cluster-nightly-5-0-5r4lh5 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001322450\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-5r4lh5\" in resourcegroup=\"rg-nightly-5-0-kcn82j\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-5r4lh5 in resource group rg-nightly-5-0-kcn82j, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-5xwj2c/cluster-nightly-5-0-9hzz55 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005304b0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-9hzz55\" in resourcegroup=\"rg-nightly-5-0-5xwj2c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-9hzz55 in resource group rg-nightly-5-0-5xwj2c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-5xwj2c/cluster-nightly-5-0-9hzz55 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005304b0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-9hzz55\" in resourcegroup=\"rg-nightly-5-0-5xwj2c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-9hzz55 in resource group rg-nightly-5-0-5xwj2c, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064790437436067840","occurred_at":"2026-06-10T19:22:56Z","signature_id":"e42d0d536bea2a2825881c2fbbdb14a1c2f3092e0a1a999ec38114dea2b89b69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064600993957416960","occurred_at":"2026-06-10T06:42:17Z","signature_id":"7769e21e8e0e25a3b28a4d26f7c03e221fa63c03e88a09e545dbbcde73dd6d18","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064601088287313920","occurred_at":"2026-06-10T06:41:04Z","signature_id":"4af98909b7cadfcd7669a6b3b2910e09cfebe2ee3b42b291d4a227cdbbd8fc4e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-nlck2h, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nlck2h\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-06-04..2026-06-10 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25"],"prior_runs_affected":7,"prior_last_seen_at":"2026-05-27T23:14:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-jm8qc8\" for cluster \"cluster-candidate-4-20-jm8qc8\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00083e120\u003e: \n    failed to create NodePool np-jm8qc8, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-jm8qc8 in resource group rg-candidate-4-20-svkpkx, error: failed waiting for nodepool=\"np-jm8qc8\" for cluster \"cluster-candidate-4-20-jm8qc8\" in resourcegroup=\"rg-candidate-4-20-svkpkx\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-jm8qc8\" for cluster \"cluster-candidate-4-20-jm8qc8\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00083e120\u003e: \n    failed to create NodePool np-jm8qc8, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-jm8qc8 in resource group rg-candidate-4-20-svkpkx, error: failed waiting for nodepool=\"np-jm8qc8\" for cluster \"cluster-candidate-4-20-jm8qc8\" in resourcegroup=\"rg-candidate-4-20-svkpkx\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00120e3f0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-nlck2h, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nlck2h\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00120e3f0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-nlck2h, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nlck2h\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064600993957416960","occurred_at":"2026-06-10T06:42:17Z","signature_id":"301cec170c53cf8d8b7e426b98a8b89509fc6fc1e1b6aa2e45b12321f95d1026","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064601088287313920","occurred_at":"2026-06-10T06:41:04Z","signature_id":"d29e8b84a68e64de80d4cef40e3b6928e3f1f55c7e4085b40d491859f17607ec","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":15,"trend_counts":[2,2,2,2,2,2,2],"trend_range":"2026-06-04..2026-06-10 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-01"],"prior_runs_affected":8,"prior_last_seen_at":"2026-06-07T23:00:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-9kjrzr at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e08900\u003e: \n    failed to create HCP cluster np-skip-minor-9kjrzr: failed starting cluster creation \"np-skip-minor-9kjrzr\" in resourcegroup=\"rg-np-skip-minor-9kjrzr-7gs6rf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9kjrzr-7gs6rf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-9kjrzr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-9kjrzr at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e08900\u003e: \n    failed to create HCP cluster np-skip-minor-9kjrzr: failed starting cluster creation \"np-skip-minor-9kjrzr\" in resourcegroup=\"rg-np-skip-minor-9kjrzr-7gs6rf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9kjrzr-7gs6rf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-9kjrzr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b9caa0\u003e: \n    failed to create HCP cluster np-dg-minor-9c7d7p: failed starting cluster creation \"np-dg-minor-9c7d7p\" in resourcegroup=\"rg-np-dg-minor-9c7d7p-rhs2s4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-9c7d7p-rhs2s4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-9c7d7p\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b9caa0\u003e: \n    failed to create HCP cluster np-dg-minor-9c7d7p: failed starting cluster creation \"np-dg-minor-9c7d7p\" in resourcegroup=\"rg-np-dg-minor-9c7d7p-rhs2s4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-9c7d7p-rhs2s4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-9c7d7p\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064845065389346816","occurred_at":"2026-06-10T23:00:01Z","signature_id":"3341dd64a3acbe4e116886eb801cee2e83a5a5e09ad4567ef88a2ee4f821f990","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064845065389346816","occurred_at":"2026-06-10T23:00:01Z","signature_id":"608ceb4d0f69da6e26c29c6c19b820dd0d0a1814d4e32993c524f767d55b406b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":13,"trend_counts":[0,0,0,0,2,3,2],"trend_range":"2026-06-04..2026-06-10 UTC","prior_windows_present":2,"prior_window_starts":["2026-05-18","2026-05-25"],"prior_runs_affected":9,"prior_last_seen_at":"2026-05-30T02:00:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064599912179306496","occurred_at":"2026-06-10T06:43:36Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064599912179306496","occurred_at":"2026-06-10T06:43:36Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"80f757ab340f6d75766a4d77caaaddd21d63ed517eb7806a91d6c90f11d6a624","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.19.33\": cannot downgrade from version 4.21.19 to 4.19.33; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-04..2026-06-10 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0e180\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-hf2g5m\" resourcegroup=\"rg-np-dg-minor-hf2g5m-8ws4v8\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-hf2g5m-8ws4v8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-hf2g5m/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.19 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0e180\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-hf2g5m\" resourcegroup=\"rg-np-dg-minor-hf2g5m-8ws4v8\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-hf2g5m-8ws4v8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-hf2g5m/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.19 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064527988531138560","occurred_at":"2026-06-10T02:00:04Z","signature_id":"746774ea21644ed376c73a98681bb6f039f642ffbe49f99ada4286dde678302f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b86f9a264db85a731f5c6fdee94b159cf165b041d6a81b701f673ae62312bfa8","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21.18\": cannot downgrade from version 4.21.19 to 4.21.18; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-04..2026-06-10 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.18\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dc8ea0\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-62m6dt\" resourcegroup=\"rg-np-downgrade-62m6dt-z875wp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-62m6dt-z875wp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-62m6dt/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": invalid node pool version 4.21.18: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": cannot downgrade from version 4.21.19 to 4.21.18\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.18\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dc8ea0\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-62m6dt\" resourcegroup=\"rg-np-downgrade-62m6dt-z875wp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-62m6dt-z875wp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-62m6dt/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": invalid node pool version 4.21.18: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": cannot downgrade from version 4.21.19 to 4.21.18\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064527988531138560","occurred_at":"2026-06-10T02:00:04Z","signature_id":"fde6f29812e4e7abde0fa8fa5483cb40c1e00e1bb8affea6418d034b87fb7d83","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}