{"meta":{"date":"2026-06-03","timezone":"UTC","generated_at":"2026-08-29T11:28:46Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2062308558187270144","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-03T23:00:50Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0515838c95373d36175368f6e353bf62d835a2e22fcc4c37cbe53785665052fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2062308558187270144","occurred_at":"2026-06-03T23:00:50Z","failed_at":"e2e","signature_id":"a2207ec4c68b515a59f0bd21a488c5facf4b849c48a96cc2d00568fe62026daf","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:195]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001110c30\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: failed to list pods in cilium-connectivity-check namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:195]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001110c30\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: failed to list pods in cilium-connectivity-check namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"55c14c3f03d3c24ad1b84097b7e2a9a3469539374181eb8ba99dd231759efddd","failure_pattern":"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: failed to list pods in cilium-connectivity-check namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would","search_query":"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: failed to list pods in cilium-connectivity-check namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline"}},{"row_id":"8cd182a70add492bad5719b9c71bc2350804d23163b8d1f224685a177e41a33c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2062308558187270144","occurred_at":"2026-06-03T23:00:50Z","failed_at":"e2e","signature_id":"8945e31bfed0eb6ac17a63ddd9d7b23a6f01bcd2b495d70054853ec65b1c8897","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d830a0\u003e: \n    failed to create HCP cluster np-dg-minor-rdcsmw: failed starting cluster creation \"np-dg-minor-rdcsmw\" in resourcegroup=\"rg-np-dg-minor-rdcsmw-5hf5zr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-rdcsmw-5hf5zr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-rdcsmw\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d830a0\u003e: \n    failed to create HCP cluster np-dg-minor-rdcsmw: failed starting cluster creation \"np-dg-minor-rdcsmw\" in resourcegroup=\"rg-np-dg-minor-rdcsmw-5hf5zr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-rdcsmw-5hf5zr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-rdcsmw\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["55c14c3f03d3c24ad1b84097b7e2a9a3469539374181eb8ba99dd231759efddd","90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2061991328811585536","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-03T02:00:17Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"154cc7901f0c637f96500caf4f19333f187434c9cbde0522280de50fd7a8e7cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2061991328811585536","occurred_at":"2026-06-03T02:00:17Z","failed_at":"e2e","signature_id":"3e3a3d4f62c5e8a64ff8b5cb02b93498aaf8a6753c80e2b6b8b3864addabc2a3","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f962bc17a3105c00a9b0bd7f4b2aca4444ece86f9a14faacfeee24cd27ebd87","failure_pattern":"failed to wait for first cluster \"\u003ccluster\u003e\" to complete creation","search_query":"context deadline exceeded"}},{"row_id":"4a2970c1d19bf5799ff3551d1688a9acd6f24f8a84ea15ca9af08e7e79aa507c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2061991328811585536","occurred_at":"2026-06-03T02:00:17Z","failed_at":"e2e","signature_id":"d2cb850ec40d121260a9e523cc6cf7a3f30518e83472195d04d803cbf6c8e6a8","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b99d80\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-2j9tl4\" resourcegroup=\"rg-np-dg-minor-2j9tl4-bbdm8h\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-2j9tl4-bbdm8h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-2j9tl4/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": invalid node pool version 4.19.32: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": cannot downgrade from version 4.21.18 to 4.19.32\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b99d80\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-2j9tl4\" resourcegroup=\"rg-np-dg-minor-2j9tl4-bbdm8h\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-2j9tl4-bbdm8h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-2j9tl4/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": invalid node pool version 4.19.32: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.32\\\": cannot downgrade from version 4.21.18 to 4.19.32\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4512b47765e0bffaed1b7f448c71dd3b27870e3b42213084f6dadb1fa1c4af46","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.19.32\": cannot downgrade from version 4.21.18 to 4.19.32; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"7e2be44bc514b474817b5fd076ae7bad4a4952e31fab7eb2ed5ccae1292ed8b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2061991328811585536","occurred_at":"2026-06-03T02:00:17Z","failed_at":"e2e","signature_id":"027f41c1b5f1fc9fe408b393353066899b0f0ccc1d39717f59290c50e31b1019","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0d060\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-9gv25x\" resourcegroup=\"rg-np-downgrade-9gv25x-zbzp5q\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-9gv25x-zbzp5q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-9gv25x/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0d060\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-9gv25x\" resourcegroup=\"rg-np-downgrade-9gv25x-zbzp5q\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-9gv25x-zbzp5q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-9gv25x/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d81a996cdf791decc51216fb7be72b9db7c9b98c36c458e851bda1b4a4b600a6","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21.17\": cannot downgrade from version 4.21.18 to 4.21.17; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4512b47765e0bffaed1b7f448c71dd3b27870e3b42213084f6dadb1fa1c4af46","6f962bc17a3105c00a9b0bd7f4b2aca4444ece86f9a14faacfeee24cd27ebd87","d81a996cdf791decc51216fb7be72b9db7c9b98c36c458e851bda1b4a4b600a6"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}