{"meta":{"date":"2026-06-07","timezone":"UTC","generated_at":"2026-08-29T08:42:16Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2063757904837611520","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-07T23:00:01Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0b6dd684aa698092e1b90fff1bf174d0f63ce1670d2b0fb8a3c2979bf4436d62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2063757904837611520","occurred_at":"2026-06-07T23:00:01Z","failed_at":"e2e","signature_id":"dd23cb4b04f496fabdef2872a77800a895e5bc9a5f8fe811094279b9781d1742","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:83]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014819e0\u003e: \n    failed to create HCP cluster ea-listg7dqqq, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listg7dqqq in resource group ea-list-89zt2g, error: failed waiting for cluster=\"ea-listg7dqqq\" in resourcegroup=\"ea-list-89zt2g\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listg7dqqq in resource group ea-list-89zt2g, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:83]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014819e0\u003e: \n    failed to create HCP cluster ea-listg7dqqq, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listg7dqqq in resource group ea-list-89zt2g, error: failed waiting for cluster=\"ea-listg7dqqq\" in resourcegroup=\"ea-list-89zt2g\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listg7dqqq in resource group ea-list-89zt2g, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listg7dqqq in resource group ea-list-89zt2g, error: failed waiting for cluster=\"ea-listg7dqqq\" in resourcegroup=\"ea-list-89zt2g\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listg7dqqq in resource group ea-list-89zt2g, error: context deadline exceeded"}},{"row_id":"3b39185d2924c889826e5e80636049cd5b86516aa980546c11631d82baf4a35b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2063757904837611520","occurred_at":"2026-06-07T23:00:01Z","failed_at":"e2e","signature_id":"002df590bcd45cc791304d90cb69cac5c15a8158d533796fa3d126470538261f","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-nxr6v8 at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001012a40\u003e: \n    failed to create HCP cluster np-skip-minor-nxr6v8: failed starting cluster creation \"np-skip-minor-nxr6v8\" in resourcegroup=\"rg-np-skip-minor-nxr6v8-m9wqh9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-nxr6v8-m9wqh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-nxr6v8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-nxr6v8 at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001012a40\u003e: \n    failed to create HCP cluster np-skip-minor-nxr6v8: failed starting cluster creation \"np-skip-minor-nxr6v8\" in resourcegroup=\"rg-np-skip-minor-nxr6v8-m9wqh9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-nxr6v8-m9wqh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-nxr6v8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d404d6e09043ba1f84680b04d1ee196e54404ed98efde22ce9eab117b4d8caaf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2063757904837611520","occurred_at":"2026-06-07T23:00:01Z","failed_at":"e2e","signature_id":"9672314d5b1e10ebe8adc66ed14732501391a1a86cd94c0892820bf60364f0ea","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00121b560\u003e: \n    failed to create HCP cluster np-dg-minor-9mfg6z: failed starting cluster creation \"np-dg-minor-9mfg6z\" in resourcegroup=\"rg-np-dg-minor-9mfg6z-hhrzlx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-9mfg6z-hhrzlx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-9mfg6z\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00121b560\u003e: \n    failed to create HCP cluster np-dg-minor-9mfg6z: failed starting cluster creation \"np-dg-minor-9mfg6z\" in resourcegroup=\"rg-np-dg-minor-9mfg6z-hhrzlx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-9mfg6z-hhrzlx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-9mfg6z\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063440880454340608","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-07T02:00:17Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0184b3c3550c5bc42f2b3e3ece16c08b6e1b3806687098eae3d89207794fab26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063440880454340608","occurred_at":"2026-06-07T02:00:17Z","failed_at":"e2e","signature_id":"8c128d68c438396a266b7642d666f9d954ae8970a87416b693f315cb425da0f4","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00168a1e0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-mvf6h.apps.aro.cilium-cluster.rxfu.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00168a1e0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-mvf6h.apps.aro.cilium-cluster.rxfu.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"c9f8ebe254136cb3e448f912688e11c02fbb0f4801d9346fefa5552d405fe035","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063440880454340608","occurred_at":"2026-06-07T02:00:17Z","failed_at":"e2e","signature_id":"3d5f8d128b33e128a0f17f07f6e8f467353910cf858466fb1942531fd86021a3","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a426a0\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-rh478b\" resourcegroup=\"rg-np-dg-minor-rh478b-w9gctr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-rh478b-w9gctr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-rh478b/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a426a0\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-rh478b\" resourcegroup=\"rg-np-dg-minor-rh478b-w9gctr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-rh478b-w9gctr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-rh478b/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"53fa6318769bb2a786556b3fac950c44b85ab8ba4f6fa60b2949a759bcc916c1","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.19.33\": cannot downgrade from version 4.21.18 to 4.19.33; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"f89f2a5845edb5a0b217ef9e561fc1eed96628bd271d8b3728d9feaf95815f82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063440880454340608","occurred_at":"2026-06-07T02:00:17Z","failed_at":"e2e","signature_id":"c71bfba50ab84c2b978e139c5bbf0380f55b365a49799aad046fe4c3dba90531","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f121c0\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-2xj4hp\" resourcegroup=\"rg-np-downgrade-2xj4hp-dpkmmj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-2xj4hp-dpkmmj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-2xj4hp/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f121c0\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-2xj4hp\" resourcegroup=\"rg-np-downgrade-2xj4hp-dpkmmj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-2xj4hp-dpkmmj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-2xj4hp/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d81a996cdf791decc51216fb7be72b9db7c9b98c36c458e851bda1b4a4b600a6","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21.17\": cannot downgrade from version 4.21.18 to 4.21.17; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["53fa6318769bb2a786556b3fac950c44b85ab8ba4f6fa60b2949a759bcc916c1","c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","d81a996cdf791decc51216fb7be72b9db7c9b98c36c458e851bda1b4a4b600a6"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}