{"meta":{"start_date":"2026-04-06","end_date":"2026-04-06","start_at":"2026-04-06T00:00:00Z","end_at":"2026-04-07T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T17:27:03Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":7,"failed_runs":6,"raw_occurrences":13,"matched_occurrences":13,"runs_affected":6},"rows":[{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":7,"runs_affected":5,"failed_runs":5,"run_impact_percent":71.42857142857143,"anchor_occurrences":7,"after_last_push_count":44,"trend_counts":[0,0,18,8,0,9,7],"trend_range":"2026-03-31..2026-04-06 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-09","2026-03-16","2026-03-30"],"prior_runs_affected":28,"prior_last_seen_at":"2026-04-05T18:04:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041279040924422144","occurred_at":"2026-04-06T22:17:03Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041274391769124864","occurred_at":"2026-04-06T21:58:34Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041271574195081216","occurred_at":"2026-04-06T21:47:22Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041215587668987904","occurred_at":"2026-04-06T18:04:54Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041074948453175296","occurred_at":"2026-04-06T08:46:03Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041074948453175296","occurred_at":"2026-04-06T08:46:03Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041074948453175296","occurred_at":"2026-04-06T08:46:03Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":33,"trend_counts":[0,0,18,6,0,3,1],"trend_range":"2026-03-31..2026-04-06 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-16","2026-03-23","2026-03-30"],"prior_runs_affected":14,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775503018 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:237\n STEP: creating resource group @ 04/06/26 19:16:58.68\n \"ts\"=\"2026-04-06 19:16:58.680121\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\"\n STEP: creating cluster parameters at control plane version @ 04/06/26 19:17:02.655\n STEP: creating customer resources @ 04/06/26 19:17:02.751\n STEP: creating the HCP cluster with version 4.20.17 @ 04/06/26 19:18:03.735\n \"ts\"=\"2026-04-06 19:18:03.735663\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-rrxpd6\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\"\n STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/06/26 19:27:58.589\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/06/26 19:37:32.137\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/06/26 19:39:39.317\n STEP: capturing node release images before upgrade @ 04/06/26 19:39:39.492\n STEP: triggering nodepool upgrade to version 4.20.17 @ 04/06/26 19:39:39.666\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/06/26 19:40:43.947\n STEP: verifying node pool GET still reflects the new version @ 04/06/26 20:08:51.474\n \"ts\"=\"2026-04-06 20:08:51.771658\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-06 20:08:51.771800\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-rrxpd6-zbqc7q\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\"\n \"ts\"=\"2026-04-06 20:08:51.932025\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-rrxpd6-zbqc7q\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-06 20:08:53.805410\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-06 20:08:53.805517\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-06 20:08:53.805583\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-06 20:08:53.807054\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-134\"\n \"ts\"=\"2026-04-06 20:08:53.807062\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\"\n \"ts\"=\"2026-04-06 20:08:53.990241\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\" \"deployment\"=\"pooledNonMsiScopedAssignments-whdkgg53trd2y\"\n \"ts\"=\"2026-04-06 20:08:54.131102\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-134\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-06 20:08:54.324615\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-134\" \"deployment\"=\"msi-deployment-134\"\n \"ts\"=\"2026-04-06 20:08:54.518036\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-rrxpd6-zbqc7q\"\n \"ts\"=\"2026-04-06 20:08:54.673903\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-06 20:08:54.674011\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-06 20:08:54.674156\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775503018 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:237\n STEP: creating resource group @ 04/06/26 19:16:58.68\n \"ts\"=\"2026-04-06 19:16:58.680121\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\"\n STEP: creating cluster parameters at control plane version @ 04/06/26 19:17:02.655\n STEP: creating customer resources @ 04/06/26 19:17:02.751\n STEP: creating the HCP cluster with version 4.20.17 @ 04/06/26 19:18:03.735\n \"ts\"=\"2026-04-06 19:18:03.735663\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-rrxpd6\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\"\n STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/06/26 19:27:58.589\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/06/26 19:37:32.137\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/06/26 19:39:39.317\n STEP: capturing node release images before upgrade @ 04/06/26 19:39:39.492\n STEP: triggering nodepool upgrade to version 4.20.17 @ 04/06/26 19:39:39.666\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/06/26 19:40:43.947\n STEP: verifying node pool GET still reflects the new version @ 04/06/26 20:08:51.474\n \"ts\"=\"2026-04-06 20:08:51.771658\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-06 20:08:51.771800\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-rrxpd6-zbqc7q\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\"\n \"ts\"=\"2026-04-06 20:08:51.932025\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-rrxpd6-zbqc7q\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-06 20:08:53.805410\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-06 20:08:53.805517\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-06 20:08:53.805583\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-06 20:08:53.807054\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-134\"\n \"ts\"=\"2026-04-06 20:08:53.807062\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\"\n \"ts\"=\"2026-04-06 20:08:53.990241\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\" \"deployment\"=\"pooledNonMsiScopedAssignments-whdkgg53trd2y\"\n \"ts\"=\"2026-04-06 20:08:54.131102\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-134\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-06 20:08:54.324615\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-134\" \"deployment\"=\"msi-deployment-134\"\n \"ts\"=\"2026-04-06 20:08:54.518036\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-rrxpd6-zbqc7q\"\n \"ts\"=\"2026-04-06 20:08:54.673903\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-06 20:08:54.674011\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-06 20:08:54.674156\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-rrxpd6-9pwrvq\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041215587668987904","occurred_at":"2026-04-06T18:04:54Z","signature_id":"5e3ed17db3d12f1ab35835192b8926fafab58511a775c141ecb3fa76b36efd69","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"21d4ef5ea099bcab4c9b17a22b87833125050f5319c2fc11c95f48e6b41e5281","failure_pattern":"ERROR CODE: Conflict; detail message Cannot update resource while resource is \"accepted\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-31..2026-04-06 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":2,"prior_last_seen_at":"2026-03-25T10:46:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0001769a0\u003e: \n failed to create HCP cluster list-test-cluster-p4ct5p: failed starting cluster creation \"list-test-cluster-p4ct5p\" in resourcegroup=\"cluster-listing-m5pqxl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster list-test-cluster-p4ct5p: failed starting cluster creation \\\"list-test-cluster-p4ct5p\\\" in resourcegroup=\\\"cluster-listing-m5pqxl\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000176980\u003e{\n msg: \"failed starting cluster creation \\\"list-test-cluster-p4ct5p\\\" in resourcegroup=\\\"cluster-listing-m5pqxl\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000c6f590\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Date\": [\n \"Mon, 06 Apr 2026 18:28:48 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/f8dcff39-92dd-4b53-bfd5-a54e320e9c99\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260406T182849Z:c7e70c50-1876-4215-b20d-f5fe6a8b0f35\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 6C605D2072634CEB987C0EC2A7E6913D Ref B: CH1AA2020603011 Ref C: 2026-04-06T18:28:45Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"057ea031-c736-4cfd-82cb-048ffa7ce574\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"c7e70c50-18...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0001769a0\u003e: \n failed to create HCP cluster list-test-cluster-p4ct5p: failed starting cluster creation \"list-test-cluster-p4ct5p\" in resourcegroup=\"cluster-listing-m5pqxl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster list-test-cluster-p4ct5p: failed starting cluster creation \\\"list-test-cluster-p4ct5p\\\" in resourcegroup=\\\"cluster-listing-m5pqxl\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000176980\u003e{\n msg: \"failed starting cluster creation \\\"list-test-cluster-p4ct5p\\\" in resourcegroup=\\\"cluster-listing-m5pqxl\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000c6f590\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Date\": [\n \"Mon, 06 Apr 2026 18:28:48 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/f8dcff39-92dd-4b53-bfd5-a54e320e9c99\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260406T182849Z:c7e70c50-1876-4215-b20d-f5fe6a8b0f35\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 6C605D2072634CEB987C0EC2A7E6913D Ref B: CH1AA2020603011 Ref C: 2026-04-06T18:28:45Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"057ea031-c736-4cfd-82cb-048ffa7ce574\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"c7e70c50-18...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041215587668987904","occurred_at":"2026-04-06T18:04:54Z","signature_id":"5421349c67ce3eae3e20d07701ef2624c120001dceeca766d2af1759bc03dd86","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":18,"trend_counts":[0,0,7,7,0,3,1],"trend_range":"2026-03-31..2026-04-06 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-30"],"prior_runs_affected":8,"prior_last_seen_at":"2026-04-05T20:06:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b98320\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-nxrhjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-nxrhjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000271d40\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"1041c110-64e3-4647-b077-e678f611e5d2\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/4e5bcd42-e79e-41df-b439-262ce47043fb\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260406T100012Z:10c71c98-ea17-4248-afb0-40a2d688a221\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"10c71c98-ea17-4248-afb0-40a2d688a221\",\n ],\n \"Content-Length\": [\"107\"],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2235\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 587DD9200B6E4171A675EA5401F200EE Ref B: CH1AA2020603035 Ref C: 2026-04-06T10:00:10Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"Date\": [\n \"Mon, 06 Apr 2026 10:00:12 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b98220\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"managem...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b98320\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-nxrhjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-nxrhjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000271d40\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"1041c110-64e3-4647-b077-e678f611e5d2\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/4e5bcd42-e79e-41df-b439-262ce47043fb\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260406T100012Z:10c71c98-ea17-4248-afb0-40a2d688a221\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"10c71c98-ea17-4248-afb0-40a2d688a221\",\n ],\n \"Content-Length\": [\"107\"],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2235\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 587DD9200B6E4171A675EA5401F200EE Ref B: CH1AA2020603035 Ref C: 2026-04-06T10:00:10Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"Date\": [\n \"Mon, 06 Apr 2026 10:00:12 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b98220\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"managem...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041074948453175296","occurred_at":"2026-04-06T08:46:03Z","signature_id":"6fd38f04ffcbf33044f9345daba9d5b99aae6f3ccf9b4ee5177e79aa64761c40","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout","failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":8,"trend_counts":[0,0,2,2,0,1,1],"trend_range":"2026-03-31..2026-04-06 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-23","2026-03-30"],"prior_runs_affected":8,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":1}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-06T20:27:11Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041215587668987904","occurred_at":"2026-04-06T18:04:54Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa2262f041e66eb099e37b23695a2256e26370a4e886b5e34fb2907105c756d4","failure_pattern":"pod \"\u003cpod\u003e\" is forbidden: error fetching namespace \"\u003cnamespace\u003e\": unable to find annotation openshift.io/sa.scc.uid-range","search_query":"pods \"wi-test-pod\" is forbidden: error fetching namespace \"e2e-oidc-wi\": unable to find annotation openshift.io/sa.scc.uid-range","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-31..2026-04-06 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:344]: Unexpected error:\n \u003c*errors.StatusError | 0xc0012b7ae0\u003e: \n pods \"wi-test-pod\" is forbidden: error fetching namespace \"e2e-oidc-wi\": unable to find annotation openshift.io/sa.scc.uid-range\n {\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods \\\"wi-test-pod\\\" is forbidden: error fetching namespace \\\"e2e-oidc-wi\\\": unable to find annotation openshift.io/sa.scc.uid-range\",\n Reason: \"Forbidden\",\n Details: {Name: \"wi-test-pod\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:344]: Unexpected error:\n \u003c*errors.StatusError | 0xc0012b7ae0\u003e: \n pods \"wi-test-pod\" is forbidden: error fetching namespace \"e2e-oidc-wi\": unable to find annotation openshift.io/sa.scc.uid-range\n {\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods \\\"wi-test-pod\\\" is forbidden: error fetching namespace \\\"e2e-oidc-wi\\\": unable to find annotation openshift.io/sa.scc.uid-range\",\n Reason: \"Forbidden\",\n Details: {Name: \"wi-test-pod\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040950439863652352","occurred_at":"2026-04-06T00:31:18Z","signature_id":"c525179ba8c84cfd1edbe946a68a62380981593d54d12a25e846f243622ca4b1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-2trfm7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-31..2026-04-06 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-16","2026-03-23","2026-03-30"],"prior_runs_affected":14,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014aaa80\u003e: \n failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-2trfm7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7, error: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-2trfm7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ef8f30\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7\",\n },\n \u003c*fmt.wrapErrors | 0xc0014aaa50\u003e{\n msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-2trfm7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ef8f30\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014aaa80\u003e: \n failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-2trfm7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7, error: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-2trfm7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ef8f30\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7\",\n },\n \u003c*fmt.wrapErrors | 0xc0014aaa50\u003e{\n msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-2trfm7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ef8f30\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-2trfm7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041274391769124864","occurred_at":"2026-04-06T21:58:34Z","signature_id":"b6a06340d62d98c098203a1aba2a3caa8ba91ba12899e2bd27a359543dee5d54","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}