{"meta":{"date":"2026-06-04","timezone":"UTC","generated_at":"2026-08-29T10:53:37Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2062670783078993920","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-04T23:00:11Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"abc28fa7d70b41925be72327fc66657759fcec86e133e66457e1da368c4cba09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2062670783078993920","occurred_at":"2026-06-04T23:00:11Z","failed_at":"e2e","signature_id":"4163876692a26a625b53e5b9a6671a9785c4d7e246164e79571775944dfd95cc","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-ppxbxk at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00173e740\u003e: \n    failed to create HCP cluster np-skip-minor-ppxbxk: failed starting cluster creation \"np-skip-minor-ppxbxk\" in resourcegroup=\"rg-np-skip-minor-ppxbxk-4xfczz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-ppxbxk-4xfczz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-ppxbxk\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-ppxbxk at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00173e740\u003e: \n    failed to create HCP cluster np-skip-minor-ppxbxk: failed starting cluster creation \"np-skip-minor-ppxbxk\" in resourcegroup=\"rg-np-skip-minor-ppxbxk-4xfczz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-ppxbxk-4xfczz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-ppxbxk\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d1de0b4b03a2b00c9fb89922674fa991c5ca97b48ebfcc8ec0f870b0e17a7f88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2062670783078993920","occurred_at":"2026-06-04T23:00:11Z","failed_at":"e2e","signature_id":"abfe0ac1179ac3c2520f4d3bb52e43988fb26a7ce5a5f115b571d5d0589b25fd","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00090c540\u003e: \n    failed to create HCP cluster np-dg-minor-9mggnq: failed starting cluster creation \"np-dg-minor-9mggnq\" in resourcegroup=\"rg-np-dg-minor-9mggnq-fdbnbv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-9mggnq-fdbnbv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-9mggnq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00090c540\u003e: \n    failed to create HCP cluster np-dg-minor-9mggnq: failed starting cluster creation \"np-dg-minor-9mggnq\" in resourcegroup=\"rg-np-dg-minor-9mggnq-fdbnbv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-9mggnq-fdbnbv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-9mggnq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2062354554908839936","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-04T02:00:51Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"3c038ae943e9d6a74ca0611f388c3892de627d0cba03c58c3176d1bf90213099","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2062354554908839936","occurred_at":"2026-06-04T02:00:51Z","failed_at":"e2e","signature_id":"770df893a3e48c572bf43f2dd03ea20f6d9fbe2d92173f538602730d3075d2ce","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00086b020\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-852bdn\" resourcegroup=\"rg-np-downgrade-852bdn-xfb479\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-852bdn-xfb479/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-852bdn/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00086b020\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-852bdn\" resourcegroup=\"rg-np-downgrade-852bdn-xfb479\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-852bdn-xfb479/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-852bdn/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d81a996cdf791decc51216fb7be72b9db7c9b98c36c458e851bda1b4a4b600a6","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21.17\": cannot downgrade from version 4.21.18 to 4.21.17; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"6bc844f09305b2a89a1bdf104204efed4fe6d3b2aebbc75da1fcc78e573735b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2062354554908839936","occurred_at":"2026-06-04T02:00:51Z","failed_at":"e2e","signature_id":"720ccd9bdedfe3644b3bb899ebf7745b5eff02899f7a560bd85f3fff5b1d514c","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:171]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001fec150\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-fwh42.apps.aro.cilium-cl.cng1.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:171]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001fec150\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-fwh42.apps.aro.cilium-cl.cng1.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"98db581af060c2841a57372193e42fc87514c176bdb930bd1d453be565542ae6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2062354554908839936","occurred_at":"2026-06-04T02:00:51Z","failed_at":"e2e","signature_id":"0febcf1aaeec5fad8db65a225474ca88fd860c058346b24c27503c37677e70ea","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dce500\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-85jlbs\" resourcegroup=\"rg-np-dg-minor-85jlbs-dnwvhj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-85jlbs-dnwvhj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-85jlbs/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": invalid node pool version 4.19.32: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": cannot downgrade from version 4.21.18 to 4.19.32\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dce500\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-85jlbs\" resourcegroup=\"rg-np-dg-minor-85jlbs-dnwvhj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-85jlbs-dnwvhj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-85jlbs/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": invalid node pool version 4.19.32: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": cannot downgrade from version 4.21.18 to 4.19.32\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4512b47765e0bffaed1b7f448c71dd3b27870e3b42213084f6dadb1fa1c4af46","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.19.32\": cannot downgrade from version 4.21.18 to 4.19.32; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4512b47765e0bffaed1b7f448c71dd3b27870e3b42213084f6dadb1fa1c4af46","c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","d81a996cdf791decc51216fb7be72b9db7c9b98c36c458e851bda1b4a4b600a6"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}