{"meta":{"start_date":"2026-06-01","end_date":"2026-06-07","start_at":"2026-06-01T00:00:00Z","end_at":"2026-06-08T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T08:25:47Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":12,"failed_runs":10,"raw_occurrences":25,"matched_occurrences":25,"runs_affected":10},"rows":[{"environment":"prod","failure_pattern_id":"90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":9,"runs_affected":5,"failed_runs":5,"run_impact_percent":41.666666666666664,"anchor_occurrences":9,"after_last_push_count":9,"trend_counts":[0,0,0,1,2,2,2,2],"trend_range":"2026-05-31..2026-06-07 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-nxr6v8 at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001012a40\u003e: \n    failed to create HCP cluster np-skip-minor-nxr6v8: failed starting cluster creation \"np-skip-minor-nxr6v8\" in resourcegroup=\"rg-np-skip-minor-nxr6v8-m9wqh9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-nxr6v8-m9wqh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-nxr6v8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-nxr6v8 at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001012a40\u003e: \n    failed to create HCP cluster np-skip-minor-nxr6v8: failed starting cluster creation \"np-skip-minor-nxr6v8\" in resourcegroup=\"rg-np-skip-minor-nxr6v8-m9wqh9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-nxr6v8-m9wqh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-nxr6v8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00121b560\u003e: \n    failed to create HCP cluster np-dg-minor-9mfg6z: failed starting cluster creation \"np-dg-minor-9mfg6z\" in resourcegroup=\"rg-np-dg-minor-9mfg6z-hhrzlx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-9mfg6z-hhrzlx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-9mfg6z\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00121b560\u003e: \n    failed to create HCP cluster np-dg-minor-9mfg6z: failed starting cluster creation \"np-dg-minor-9mfg6z\" in resourcegroup=\"rg-np-dg-minor-9mfg6z-hhrzlx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-9mfg6z-hhrzlx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-9mfg6z\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-nt5h8p at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d69860\u003e: \n    failed to create HCP cluster np-skip-minor-nt5h8p: failed starting cluster creation \"np-skip-minor-nt5h8p\" in resourcegroup=\"rg-np-skip-minor-nt5h8p-4zk5s6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-nt5h8p-4zk5s6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-nt5h8p\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-nt5h8p at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d69860\u003e: \n    failed to create HCP cluster np-skip-minor-nt5h8p: failed starting cluster creation \"np-skip-minor-nt5h8p\" in resourcegroup=\"rg-np-skip-minor-nt5h8p-4zk5s6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-nt5h8p-4zk5s6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-nt5h8p\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2063757904837611520","occurred_at":"2026-06-07T23:00:01Z","signature_id":"002df590bcd45cc791304d90cb69cac5c15a8158d533796fa3d126470538261f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2063757904837611520","occurred_at":"2026-06-07T23:00:01Z","signature_id":"9672314d5b1e10ebe8adc66ed14732501391a1a86cd94c0892820bf60364f0ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2063395653450469376","occurred_at":"2026-06-06T23:00:16Z","signature_id":"b19fc78175ae6a4d76bfa96e0223d47371a3f8190b514321de07ef29a66c8e3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2063395653450469376","occurred_at":"2026-06-06T23:00:16Z","signature_id":"efebc2dd954574e86c7345f4f3673df28db12bc7f40007b37b44f7f3daccb2dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2063033187675672576","occurred_at":"2026-06-05T23:00:15Z","signature_id":"067b1065bfc12ff3f4e5d6501a06ba82da6ac01555371bf4e6db7e2b25036a8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2063033187675672576","occurred_at":"2026-06-05T23:00:15Z","signature_id":"8acc9dad3e2c95defd2442d8f070841e9aee33305d4b51957fd8161c356d0087","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2062670783078993920","occurred_at":"2026-06-04T23:00:11Z","signature_id":"4163876692a26a625b53e5b9a6671a9785c4d7e246164e79571775944dfd95cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2062670783078993920","occurred_at":"2026-06-04T23:00:11Z","signature_id":"abfe0ac1179ac3c2520f4d3bb52e43988fb26a7ce5a5f115b571d5d0589b25fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2062308558187270144","occurred_at":"2026-06-03T23:00:50Z","signature_id":"8945e31bfed0eb6ac17a63ddd9d7b23a6f01bcd2b495d70054853ec65b1c8897","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d81a996cdf791decc51216fb7be72b9db7c9b98c36c458e851bda1b4a4b600a6","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21.17\": cannot downgrade from version 4.21.18 to 4.21.17; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":41.666666666666664,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,1,1,1,1,1],"trend_range":"2026-05-31..2026-06-07 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f121c0\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-2xj4hp\" resourcegroup=\"rg-np-downgrade-2xj4hp-dpkmmj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-2xj4hp-dpkmmj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-2xj4hp/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f121c0\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-2xj4hp\" resourcegroup=\"rg-np-downgrade-2xj4hp-dpkmmj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-2xj4hp-dpkmmj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-2xj4hp/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a21480\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-vzj76h\" resourcegroup=\"rg-np-downgrade-vzj76h-r6prv6\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-vzj76h-r6prv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-vzj76h/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a21480\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-vzj76h\" resourcegroup=\"rg-np-downgrade-vzj76h-r6prv6\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-vzj76h-r6prv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-vzj76h/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00045dcc0\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-4z9hsj\" resourcegroup=\"rg-np-downgrade-4z9hsj-qn6gtn\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-4z9hsj-qn6gtn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-4z9hsj/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00045dcc0\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-4z9hsj\" resourcegroup=\"rg-np-downgrade-4z9hsj-qn6gtn\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-4z9hsj-qn6gtn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-4z9hsj/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063440880454340608","occurred_at":"2026-06-07T02:00:17Z","signature_id":"c71bfba50ab84c2b978e139c5bbf0380f55b365a49799aad046fe4c3dba90531","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063078486624440320","occurred_at":"2026-06-06T02:00:15Z","signature_id":"eadad8b3d51ecc61d1cce5343ee5d70b2ed046a13c80773b1b98f8cf2d4b61df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2062716081771909120","occurred_at":"2026-06-05T02:00:11Z","signature_id":"29232ad8729f8e33d05212ac0f433fe3f6cb1bc6fe2aeb27cb8a198521493be6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2062354554908839936","occurred_at":"2026-06-04T02:00:51Z","signature_id":"770df893a3e48c572bf43f2dd03ea20f6d9fbe2d92173f538602730d3075d2ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2061991328811585536","occurred_at":"2026-06-03T02:00:17Z","signature_id":"027f41c1b5f1fc9fe408b393353066899b0f0ccc1d39717f59290c50e31b1019","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"53fa6318769bb2a786556b3fac950c44b85ab8ba4f6fa60b2949a759bcc916c1","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.19.33\": cannot downgrade from version 4.21.18 to 4.19.33; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":25,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,1,1],"trend_range":"2026-05-31..2026-06-07 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a426a0\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-rh478b\" resourcegroup=\"rg-np-dg-minor-rh478b-w9gctr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-rh478b-w9gctr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-rh478b/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a426a0\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-rh478b\" resourcegroup=\"rg-np-dg-minor-rh478b-w9gctr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-rh478b-w9gctr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-rh478b/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e0f9a0\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-gqt6vk\" resourcegroup=\"rg-np-dg-minor-gqt6vk-2nr5sk\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-gqt6vk-2nr5sk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-gqt6vk/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e0f9a0\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-gqt6vk\" resourcegroup=\"rg-np-dg-minor-gqt6vk-2nr5sk\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-gqt6vk-2nr5sk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-gqt6vk/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d5d120\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-thcd8z\" resourcegroup=\"rg-np-dg-minor-thcd8z-w24xx4\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-thcd8z-w24xx4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-thcd8z/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d5d120\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-thcd8z\" resourcegroup=\"rg-np-dg-minor-thcd8z-w24xx4\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-thcd8z-w24xx4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-thcd8z/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063440880454340608","occurred_at":"2026-06-07T02:00:17Z","signature_id":"3d5f8d128b33e128a0f17f07f6e8f467353910cf858466fb1942531fd86021a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063078486624440320","occurred_at":"2026-06-06T02:00:15Z","signature_id":"ad1f2caf328f4cf617d7571c9652e0902a8afadbafad407b75e3342a68ad5f85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2062716081771909120","occurred_at":"2026-06-05T02:00:11Z","signature_id":"61dfd4c881accf81e3a7376cf1cc9c293aa3bc61d3c2841f7f28e17797a04a70","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4512b47765e0bffaed1b7f448c71dd3b27870e3b42213084f6dadb1fa1c4af46","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.19.32\": cannot downgrade from version 4.21.18 to 4.19.32; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,1,1,0,0,0],"trend_range":"2026-05-31..2026-06-07 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dce500\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-85jlbs\" resourcegroup=\"rg-np-dg-minor-85jlbs-dnwvhj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-85jlbs-dnwvhj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-85jlbs/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": invalid node pool version 4.19.32: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": cannot downgrade from version 4.21.18 to 4.19.32\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dce500\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-85jlbs\" resourcegroup=\"rg-np-dg-minor-85jlbs-dnwvhj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-85jlbs-dnwvhj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-85jlbs/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": invalid node pool version 4.19.32: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": cannot downgrade from version 4.21.18 to 4.19.32\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b99d80\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-2j9tl4\" resourcegroup=\"rg-np-dg-minor-2j9tl4-bbdm8h\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-2j9tl4-bbdm8h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-2j9tl4/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": invalid node pool version 4.19.32: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": cannot downgrade from version 4.21.18 to 4.19.32\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b99d80\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-2j9tl4\" resourcegroup=\"rg-np-dg-minor-2j9tl4-bbdm8h\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-2j9tl4-bbdm8h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-2j9tl4/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": invalid node pool version 4.19.32: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": cannot downgrade from version 4.21.18 to 4.19.32\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2062354554908839936","occurred_at":"2026-06-04T02:00:51Z","signature_id":"0febcf1aaeec5fad8db65a225474ca88fd860c058346b24c27503c37677e70ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2061991328811585536","occurred_at":"2026-06-03T02:00:17Z","signature_id":"d2cb850ec40d121260a9e523cc6cf7a3f30518e83472195d04d803cbf6c8e6a8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":6,"trend_counts":[0,0,0,0,1,0,0,1],"trend_range":"2026-05-31..2026-06-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-05-11","2026-05-18"],"prior_runs_affected":6,"prior_last_seen_at":"2026-05-20T20:02:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00168a1e0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-mvf6h.apps.aro.cilium-cluster.rxfu.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00168a1e0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-mvf6h.apps.aro.cilium-cluster.rxfu.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:171]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001fec150\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-fwh42.apps.aro.cilium-cl.cng1.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:171]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001fec150\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-fwh42.apps.aro.cilium-cl.cng1.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063440880454340608","occurred_at":"2026-06-07T02:00:17Z","signature_id":"8c128d68c438396a266b7642d666f9d954ae8970a87416b693f315cb425da0f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2062354554908839936","occurred_at":"2026-06-04T02:00:51Z","signature_id":"720ccd9bdedfe3644b3bb899ebf7745b5eff02899f7a560bd85f3fff5b1d514c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":1,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-05-31..2026-06-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-04","2026-05-11","2026-05-18"],"prior_runs_affected":5,"prior_last_seen_at":"2026-05-21T03:19:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["30 pc=0x6b4e210\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f40654cc128?, 0xc00148eb70?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc000cdd8b0 sp=0xc000cdd888 pc=0x6b69d33\nruntime.call16(0xc000a17590, 0xc000e04410, 0x0, 0x0, 0x0, 0x10, 0xc000cddb08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc000cdd8d0 sp=0xc000cdd8b0 pc=0x2ed1223\nruntime.reflectcall(0x7ff2360?, 0xc00148eb70?, 0x2?, 0x86532af?, 0x0?, 0x12?, 0x7bb8920?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc000cdd910 sp=0xc000cdd8d0 pc=0x2ed9ab6\nreflect.Value.call({0x746e4e0?, 0xc000e04410?, 0xc000132ed8?}, {0x860f1d5, 0x4}, {0xc000132f60, 0x1, 0x0?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc000cdde70 sp=0xc000cdd910 pc=0x2f0cc66\nreflect.Value.Call({0x746e4e0?, 0xc000e04410?, 0x0?}, {0xc000132f60?, 0x2eef2c9?, 0x6b4f6da?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc000cddee8 sp=0xc000cdde70 pc=0x2f0bd59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x91f9670?, 0xc00148eb70?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc000cddf88 sp=0xc000cddee8 pc=0x3330170\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000cddfe0 sp=0xc000cddf88 pc=0x3342c3b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000cddfe8 sp=0xc000cddfe0 pc=0x2ed2cc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 98\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 434 gp=0xc001582a80 m=nil [select]:\nruntime.gopark(0xc000091768?, 0x2?, 0x60?, 0xc8?, 0xc00009175c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0000915e8 sp=0xc0000915c8 pc=0x2eca50e\nruntime.selectgo(0xc000091768, 0xc000091758, 0xc000ff47a0?, 0x0, 0x3698d5c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000091728 sp=0xc0000915e8 pc=0x2ea52f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x91df3b8, 0xc0005abc00}, 0x863299e?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc000091798 sp=0xc000091728 pc=0x36984b6\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x9230900, {0x91df3b8, 0xc0005abc00}, 0x2ec7af3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc000091a98 sp=0xc000091798 pc=0x3d99192\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x91df3b8, 0xc0005abc00}, 0xc0018106a8, {0xc000082948, 0x18}, {0xc00065cb60, 0x1d})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:239 +0x365 fp=0xc000091c20 sp=0xc000091a98 pc=0x6b2e305\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x91df3b8?, 0xc0005abc00?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:197 +0x72 fp=0xc000091cc8 sp=0xc000091c20 pc=0x6b2dd32\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x91df3b8?, 0xc0005abc00?}, 0xc0005abc10?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc000091d20 sp=0xc000091cc8 pc=0x364ba82\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x91df3b8, 0xc0005abc00}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc000091e68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc000091d98 sp=0xc000091d20 pc=0x3649b2e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x91df348, 0xc0007c4730}, 0xc0018106a8, {0xc000082948, 0x18}, {0xc00065cb60, 0x1d}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:195 +0x285 fp=0xc000091ef0 sp=0xc000091d98 pc=0x6b2daa5\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:521 +0x7e fp=0xc000091f78 sp=0xc000091ef0 pc=0x6b3199e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc000091fe0 sp=0xc000091f78 pc=0x4766a30\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000091fe8 sp=0xc000091fe0 pc=0x2ed2cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 395\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xfffffffffffffffc\nrbx    0x5\nrcx    0x2e49e2e\nrdx    0x80\nrdi    0x5\nrsi    0x7ffe401a3a54\nrbp    0x7ffe401a3a08\nrsp    0x7ffe401a39c0\nr8     0x0\nr9     0x0\nr10    0x22\nr11    0x246\nr12    0x7ffe401a3a98\nr13    0x1\nr14    0xc7ea580\nr15    0x1\nrip    0x2e49e2e\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\n30 pc=0x6b4e210\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f40654cc128?, 0xc00148eb70?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc000cdd8b0 sp=0xc000cdd888 pc=0x6b69d33\nruntime.call16(0xc000a17590, 0xc000e04410, 0x0, 0x0, 0x0, 0x10, 0xc000cddb08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc000cdd8d0 sp=0xc000cdd8b0 pc=0x2ed1223\nruntime.reflectcall(0x7ff2360?, 0xc00148eb70?, 0x2?, 0x86532af?, 0x0?, 0x12?, 0x7bb8920?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc000cdd910 sp=0xc000cdd8d0 pc=0x2ed9ab6\nreflect.Value.call({0x746e4e0?, 0xc000e04410?, 0xc000132ed8?}, {0x860f1d5, 0x4}, {0xc000132f60, 0x1, 0x0?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc000cdde70 sp=0xc000cdd910 pc=0x2f0cc66\nreflect.Value.Call({0x746e4e0?, 0xc000e04410?, 0x0?}, {0xc000132f60?, 0x2eef2c9?, 0x6b4f6da?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc000cddee8 sp=0xc000cdde70 pc=0x2f0bd59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x91f9670?, 0xc00148eb70?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc000cddf88 sp=0xc000cddee8 pc=0x3330170\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000cddfe0 sp=0xc000cddf88 pc=0x3342c3b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000cddfe8 sp=0xc000cddfe0 pc=0x2ed2cc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 98\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 434 gp=0xc001582a80 m=nil [select]:\nruntime.gopark(0xc000091768?, 0x2?, 0x60?, 0xc8?, 0xc00009175c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0000915e8 sp=0xc0000915c8 pc=0x2eca50e\nruntime.selectgo(0xc000091768, 0xc000091758, 0xc000ff47a0?, 0x0, 0x3698d5c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000091728 sp=0xc0000915e8 pc=0x2ea52f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x91df3b8, 0xc0005abc00}, 0x863299e?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc000091798 sp=0xc000091728 pc=0x36984b6\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x9230900, {0x91df3b8, 0xc0005abc00}, 0x2ec7af3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc000091a98 sp=0xc000091798 pc=0x3d99192\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x91df3b8, 0xc0005abc00}, 0xc0018106a8, {0xc000082948, 0x18}, {0xc00065cb60, 0x1d})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:239 +0x365 fp=0xc000091c20 sp=0xc000091a98 pc=0x6b2e305\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x91df3b8?, 0xc0005abc00?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:197 +0x72 fp=0xc000091cc8 sp=0xc000091c20 pc=0x6b2dd32\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x91df3b8?, 0xc0005abc00?}, 0xc0005abc10?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc000091d20 sp=0xc000091cc8 pc=0x364ba82\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x91df3b8, 0xc0005abc00}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc000091e68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc000091d98 sp=0xc000091d20 pc=0x3649b2e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x91df348, 0xc0007c4730}, 0xc0018106a8, {0xc000082948, 0x18}, {0xc00065cb60, 0x1d}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:195 +0x285 fp=0xc000091ef0 sp=0xc000091d98 pc=0x6b2daa5\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:521 +0x7e fp=0xc000091f78 sp=0xc000091ef0 pc=0x6b3199e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc000091fe0 sp=0xc000091f78 pc=0x4766a30\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000091fe8 sp=0xc000091fe0 pc=0x2ed2cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 395\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xfffffffffffffffc\nrbx    0x5\nrcx    0x2e49e2e\nrdx    0x80\nrdi    0x5\nrsi    0x7ffe401a3a54\nrbp    0x7ffe401a3a08\nrsp    0x7ffe401a39c0\nr8     0x0\nr9     0x0\nr10    0x22\nr11    0x246\nr12    0x7ffe401a3a98\nr13    0x1\nr14    0xc7ea580\nr15    0x1\nrip    0x2e49e2e\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2062716081771909120","occurred_at":"2026-06-05T02:00:11Z","signature_id":"bd0fa9437200809ebcafed7311e802bda04e5e699037e02405a02bda068d8b75","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"55c14c3f03d3c24ad1b84097b7e2a9a3469539374181eb8ba99dd231759efddd","failure_pattern":"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: failed to list pods in cilium-connectivity-check namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would","search_query":"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: failed to list pods in cilium-connectivity-check namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-05-31..2026-06-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:195]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001110c30\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: failed to list pods in cilium-connectivity-check namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:195]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001110c30\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: failed to list pods in cilium-connectivity-check namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2062308558187270144","occurred_at":"2026-06-03T23:00:50Z","signature_id":"a2207ec4c68b515a59f0bd21a488c5facf4b849c48a96cc2d00568fe62026daf","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6f962bc17a3105c00a9b0bd7f4b2aca4444ece86f9a14faacfeee24cd27ebd87","failure_pattern":"failed to wait for first cluster \"\u003ccluster\u003e\" to complete creation","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-05-31..2026-06-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-25"],"prior_runs_affected":2,"prior_last_seen_at":"2026-05-30T02:00:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2061991328811585536","occurred_at":"2026-06-03T02:00:17Z","signature_id":"3e3a3d4f62c5e8a64ff8b5cb02b93498aaf8a6753c80e2b6b8b3864addabc2a3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listg7dqqq in resource group ea-list-89zt2g, error: failed waiting for cluster=\"ea-listg7dqqq\" in resourcegroup=\"ea-list-89zt2g\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listg7dqqq in resource group ea-list-89zt2g, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-05-31..2026-06-07 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-04","2026-05-11","2026-05-18","2026-05-25"],"prior_runs_affected":24,"prior_last_seen_at":"2026-05-30T02:00:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:83]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014819e0\u003e: \n    failed to create HCP cluster ea-listg7dqqq, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listg7dqqq in resource group ea-list-89zt2g, error: failed waiting for cluster=\"ea-listg7dqqq\" in resourcegroup=\"ea-list-89zt2g\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listg7dqqq in resource group ea-list-89zt2g, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:83]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014819e0\u003e: \n    failed to create HCP cluster ea-listg7dqqq, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listg7dqqq in resource group ea-list-89zt2g, error: failed waiting for cluster=\"ea-listg7dqqq\" in resourcegroup=\"ea-list-89zt2g\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listg7dqqq in resource group ea-list-89zt2g, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2063757904837611520","occurred_at":"2026-06-07T23:00:01Z","signature_id":"dd23cb4b04f496fabdef2872a77800a895e5bc9a5f8fe811094279b9781d1742","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}