{"meta":{"start_date":"2026-04-07","end_date":"2026-04-07","start_at":"2026-04-07T00:00:00Z","end_at":"2026-04-08T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T18:15:50Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":6,"failed_runs":6,"raw_occurrences":61,"matched_occurrences":61,"runs_affected":6},"rows":[{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","occurrences":16,"runs_affected":6,"failed_runs":6,"run_impact_percent":100,"anchor_occurrences":16,"after_last_push_count":49,"trend_counts":[0,18,6,0,3,1,16],"trend_range":"2026-04-01..2026-04-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-16","2026-03-23","2026-03-30"],"prior_runs_affected":20,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["04/07/26 05:38:04.367\n    STEP: attempting to update nodepool version to higher than cluster version @ 04/07/26 05:48:19.896\n  \"ts\"=\"2026-04-07 05:48:20.460472\" \"msg\"=\"invalid node pool version update\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l57kvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: \\\\\\\"4.21.0\\\\\\\": invalid node pool version 4.21.0: cannot exceed control plane version 4.20.16\\\",\\n    \\\"target\\\": \\\"properties.version.id\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n    STEP: attempting to update immutable platform profile fields @ 04/07/26 05:48:20.46\n  \"ts\"=\"2026-04-07 05:48:21.151220\" \"msg\"=\"updating immutable fields\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l57kvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"4 errors occurred\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.vmSize\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.osDisk\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.availabilityZone\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  \"ts\"=\"2026-04-07 05:48:21.271777\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-07 05:48:21.271880\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-rv6wgd\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n  \"ts\"=\"2026-04-07 05:48:21.346235\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-rv6wgd\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-07 05:48:22.942477\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n  \"ts\"=\"2026-04-07 05:48:22.942620\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n  \"ts\"=\"2026-04-07 05:48:22.942678\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-07 05:48:22.944144\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-13\"\n  \"ts\"=\"2026-04-07 05:48:22.944161\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n  \"ts\"=\"2026-04-07 05:48:23.204714\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n  \"ts\"=\"2026-04-07 05:48:23.305309\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\" \"deployment\"=\"customer-infra-negative-tests-cluster-rv6wgd\"\n  \"ts\"=\"2026-04-07 05:48:23.320326\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-13\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-07 05:48:23.825441\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-13\" \"deployment\"=\"msi-deployment-13\"\n  \"ts\"=\"2026-04-07 05:48:24.341336\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-07 05:48:24.341454\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-07 05:48:24.341626\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n  \"ts\"=\"2026-04-07 06:07:46.163294\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n  \"ts\"=\"2026-04-07 06:07:46.163320\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n04/07/26 05:38:04.367\n    STEP: attempting to update nodepool version to higher than cluster version @ 04/07/26 05:48:19.896\n  \"ts\"=\"2026-04-07 05:48:20.460472\" \"msg\"=\"invalid node pool version update\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l57kvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: \\\\\\\"4.21.0\\\\\\\": invalid node pool version 4.21.0: cannot exceed control plane version 4.20.16\\\",\\n    \\\"target\\\": \\\"properties.version.id\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n    STEP: attempting to update immutable platform profile fields @ 04/07/26 05:48:20.46\n  \"ts\"=\"2026-04-07 05:48:21.151220\" \"msg\"=\"updating immutable fields\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l57kvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"4 errors occurred\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.vmSize\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.osDisk\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.availabilityZone\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  \"ts\"=\"2026-04-07 05:48:21.271777\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-07 05:48:21.271880\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-rv6wgd\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n  \"ts\"=\"2026-04-07 05:48:21.346235\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-rv6wgd\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-07 05:48:22.942477\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n  \"ts\"=\"2026-04-07 05:48:22.942620\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n  \"ts\"=\"2026-04-07 05:48:22.942678\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-07 05:48:22.944144\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-13\"\n  \"ts\"=\"2026-04-07 05:48:22.944161\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n  \"ts\"=\"2026-04-07 05:48:23.204714\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n  \"ts\"=\"2026-04-07 05:48:23.305309\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\" \"deployment\"=\"customer-infra-negative-tests-cluster-rv6wgd\"\n  \"ts\"=\"2026-04-07 05:48:23.320326\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-13\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-07 05:48:23.825441\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-13\" \"deployment\"=\"msi-deployment-13\"\n  \"ts\"=\"2026-04-07 05:48:24.341336\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-07 05:48:24.341454\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-07 05:48:24.341626\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n  \"ts\"=\"2026-04-07 06:07:46.163294\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n  \"ts\"=\"2026-04-07 06:07:46.163320\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775539371 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n    STEP: creating resource group @ 04/07/26 05:23:00.777\n  \"ts\"=\"2026-04-07 05:23:00.777205\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-9c578f-xr8n8x\"\n    STEP: creating cluster parameters at control plane version @ 04/07/26 05:23:04.384\n    STEP: creating customer resources @ 04/07/26 05:23:04.465\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 05:23:51.414\n  \"ts\"=\"2026-04-07 05:23:51.414251\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-9c578f\" \"resourceGroup\"=\"rg-np-version-upgrade-9c578f-xr8n8x\"\n    STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 05:34:55.888\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/07/26 05:45:52.841\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/07/26 05:48:21.331\n    STEP: capturing node release images before upgrade @ 04/07/26 05:48:21.463\n    STEP: triggering nodepool upgrade to version 4.21.7 @ 04/07/26 05:48:21.503\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/07/26 05:49:13.715\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775539371 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n    STEP: creating resource group @ 04/07/26 05:23:00.777\n  \"ts\"=\"2026-04-07 05:23:00.777205\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-9c578f-xr8n8x\"\n    STEP: creating cluster parameters at control plane version @ 04/07/26 05:23:04.384\n    STEP: creating customer resources @ 04/07/26 05:23:04.465\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 05:23:51.414\n  \"ts\"=\"2026-04-07 05:23:51.414251\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-9c578f\" \"resourceGroup\"=\"rg-np-version-upgrade-9c578f-xr8n8x\"\n    STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 05:34:55.888\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/07/26 05:45:52.841\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/07/26 05:48:21.331\n    STEP: capturing node release images before upgrade @ 04/07/26 05:48:21.463\n    STEP: triggering nodepool upgrade to version 4.21.7 @ 04/07/26 05:48:21.503\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/07/26 05:49:13.715\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775539380 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:234\n    STEP: creating resource group @ 04/07/26 05:23:01.232\n  \"ts\"=\"2026-04-07 05:23:01.232866\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-tczjgc-tp2rpx\"\n    STEP: creating cluster parameters at control plane version @ 04/07/26 05:23:05.133\n    STEP: creating customer resources @ 04/07/26 05:23:05.225\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 05:23:50.322\n  \"ts\"=\"2026-04-07 05:23:50.322296\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-tczjgc\" \"resourceGroup\"=\"rg-np-version-upgrade-tczjgc-tp2rpx\"\n    STEP: creating nodepool with version 4.21.5 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 05:34:54.824\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775539380 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:234\n    STEP: creating resource group @ 04/07/26 05:23:01.232\n  \"ts\"=\"2026-04-07 05:23:01.232866\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-tczjgc-tp2rpx\"\n    STEP: creating cluster parameters at control plane version @ 04/07/26 05:23:05.133\n    STEP: creating customer resources @ 04/07/26 05:23:05.225\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 05:23:50.322\n  \"ts\"=\"2026-04-07 05:23:50.322296\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-tczjgc\" \"resourceGroup\"=\"rg-np-version-upgrade-tczjgc-tp2rpx\"\n    STEP: creating nodepool with version 4.21.5 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 05:34:54.824\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"342fd958a98f57fdbc69171f73b9979d6370fb082d12515c8e5436854e410338","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"4f4470447da4e0eec3b1263376a6f4e4eabd73562a0c2787841c7a1fb3de6fef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"bc16479542f5518f199cdf90d49056dbd514a7388517335dffc90cafedff085b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"ccbd3aa0996ca39ea8f78950f49effcec35d7962ca56c70dc03209ef9115be87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","occurred_at":"2026-04-07T04:05:16Z","signature_id":"500d409f3d8fb29df0602cb09997135f46bcf75fd9f7fb14e42d1ce74f68449c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","occurred_at":"2026-04-07T04:05:16Z","signature_id":"80425a41a776bc9c054fe74e89b94fff4fb1f63049b7a42543353153422bbf16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","occurred_at":"2026-04-07T04:04:40Z","signature_id":"771a9b202a2b5fcd9bc2f6284c90ce9ab0922fc86ee71395d36f43a8a781d0d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"0eb17ac0fa73cbaf6bfd96b147f1e87899b4eb9122843f96122b57a919ecf965","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"49ed41086fe7a9ec2b601ce88c197476ebb461138a74a4df1c55c826a052fa0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"545f07c5de94eb876fe5c60d1fd48aabe80ce9fe76291be55b34245f3cddfa0e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"84b0d7cc081dfd6750b672d63540a59e19f9f30758d835bf5f5ef823dbac3aa5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"a7a3a791a16cdc30955425371d11ec180f558068ee2777f1b760ec0eec8c71db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"ea06de1114c3d8f9ae2bd132acedc5c7fdc738b85c2b59230a4f78070b8f7429","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"f1e615836088d3a70d52b7c6f718a74bd4b874c9412a0b8713b7bb1e5df7bace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"b7d39ccaa33471c6fe6d2423539073913dae6879ccbe030526e8f6b3ab342f0b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308037523705856","occurred_at":"2026-04-07T00:12:16Z","signature_id":"ab4eb11ba454086e3a61bca639145624e40a3a0c90bd62c730548e8ec4d56dc9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":14,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":14,"after_last_push_count":14,"trend_counts":[0,0,0,0,0,0,14],"trend_range":"2026-04-01..2026-04-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-23","2026-03-30"],"prior_runs_affected":11,"prior_last_seen_at":"2026-04-03T18:45:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00152e090\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-w2k2l9\" in resourcegroup=\"admin-credential-lifecycle-test-xnk8kd\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-w2k2l9 in resource group admin-credential-lifecycle-test-xnk8kd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-w2k2l9\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-xnk8kd\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-w2k2l9 in resource group admin-credential-lifecycle-test-xnk8kd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000dd89c0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-w2k2l9 in resource group admin-credential-lifecycle-test-xnk8kd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00152e090\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-w2k2l9\" in resourcegroup=\"admin-credential-lifecycle-test-xnk8kd\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-w2k2l9 in resource group admin-credential-lifecycle-test-xnk8kd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-w2k2l9\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-xnk8kd\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-w2k2l9 in resource group admin-credential-lifecycle-test-xnk8kd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000dd89c0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-w2k2l9 in resource group admin-credential-lifecycle-test-xnk8kd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012d06c0\u003e: \n    failed waiting for hcpCluster=\"cluster-stable-4-20-5875bd\" in resourcegroup=\"rg-stable-4-20-sx464q\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5875bd in resource group rg-stable-4-20-sx464q, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-5875bd\\\" in resourcegroup=\\\"rg-stable-4-20-sx464q\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5875bd in resource group rg-stable-4-20-sx464q, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001362380\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5875bd in resource group rg-stable-4-20-sx464q\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012d06c0\u003e: \n    failed waiting for hcpCluster=\"cluster-stable-4-20-5875bd\" in resourcegroup=\"rg-stable-4-20-sx464q\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5875bd in resource group rg-stable-4-20-sx464q, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-5875bd\\\" in resourcegroup=\\\"rg-stable-4-20-sx464q\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5875bd in resource group rg-stable-4-20-sx464q, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001362380\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5875bd in resource group rg-stable-4-20-sx464q\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014db050\u003e: \n    failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bqjn7p\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-bqjn7p, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-bqjn7p\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-bqjn7p, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000614870\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-bqjn7p\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014db050\u003e: \n    failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bqjn7p\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-bqjn7p, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-bqjn7p\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-bqjn7p, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000614870\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-bqjn7p\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"1afd98ace371047c0b6f30472486f90f36627888de8ef560568f39a8a9f7eefe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"34c8538cd1cf99db24364475fbc228e23e26d2362e2a83d8f0fdfb52ba4cf1f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"38826b533269d7a9bb238361448ad259ba2286646e96c6502152a76dcb1076fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"5e6c3c5c47b005967ec25cd98cba3a5126b257a4788d48b77e5a129547e95eac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"6d1fba2228413ee88801c71b444cc9f8d5620d8f429f71bb349c575fb7b5f3b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"db21d7b4d2180dfb67585d94a9c18b9307efdc3021536e3fe62d1bfcb1343968","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"e3bf0e48b6a7ad4445c218645ad83a075e948fb4e051c71c17cf9d0ae2652300","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"203e2c7ffff61cb7e8a9241ca3de6134343765d2bd071522facb4429605fa8c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"23cb7cad2d50cb707def47ef79150e901e0494d1b2b7151fbbaefbb3e6bea082","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"5be661969e52e9a052ab0134a9f3bdb6e4e82b3b6b5d2c04656940f3faa90b53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"697cb4e2d08f75c3268c0a07e907dadf1330b74f8378572502578afac4409d4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"8a59fc555efc0c6395952c408b943ce13eb2659f4641992e1435b4f7478938ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"becaebe81abc8fbb81aaadb376c5d1d5b254498b9ea945ec6a9e215bb1ffaa52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"fce7f9cb30c215532540321a2400b35a1f573a9281d5e92cf3123f1c7f7dacda","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":10,"after_last_push_count":10,"trend_counts":[0,0,0,0,0,0,10],"trend_range":"2026-04-01..2026-04-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-16","2026-03-23","2026-03-30"],"prior_runs_affected":15,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001193290\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-879vc5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-879vc5, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-879vc5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-879vc5, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001180090\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-879vc5\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001193290\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-879vc5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-879vc5, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-879vc5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-879vc5, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001180090\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-879vc5\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014d1560\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-s2sjhd, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: failed waiting for cluster=\"cluster-zstream-4-20-s2sjhd\" in resourcegroup=\"rg-zstream-upgrade-4-20-tjt8qk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-s2sjhd, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: failed waiting for cluster=\\\"cluster-zstream-4-20-s2sjhd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tjt8qk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0013b6770\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0014d1530\u003e{\n                msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-s2sjhd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tjt8qk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0013b6770\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014d1560\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-s2sjhd, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: failed waiting for cluster=\"cluster-zstream-4-20-s2sjhd\" in resourcegroup=\"rg-zstream-upgrade-4-20-tjt8qk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-s2sjhd, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: failed waiting for cluster=\\\"cluster-zstream-4-20-s2sjhd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tjt8qk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0013b6770\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0014d1530\u003e{\n                msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-s2sjhd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tjt8qk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0013b6770\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008d4210\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-jwszks\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jwszks\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000e02710\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0008d41e0\u003e{\n                msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jwszks\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000e02710\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008d4210\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-jwszks\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jwszks\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000e02710\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0008d41e0\u003e{\n                msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jwszks\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000e02710\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"19b8b89f21ef63dfdd459a5ae0dde3ec9e67b8586ab0532fdd1a4d72e5dd9677","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"3298511640d7e93d00a720ae5553b54712470ec9b619dda50523cf8dd9b743df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"45549f3d568ceae199450b06a0f29ca707fdcd572e216b08b615100bc902a118","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"48d66cb903bed78ecc5b3870134f6236fde8da937c87322b0e8aea6adb477cb3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"490a38b6630e9d08af57dc89b154f250a304afaf6ce6fbdf3395ae1e55a94afd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"4e32fb5d7ee25f5f0a0bef0524e92b9066de0d63ef1fec9063e4350475d0eb88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"513321780d22667c85d39965dcd485e1739ec694dc7dfb6d2311d111c6478061","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"5a84ec606cf94e594e22c62e9fe85fd628da238028d88ecf1ca775367060da89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"bc0af7ca1eb571869bcfd45d0933b754212864c3ca865d87bc33745cfdc01aa5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"cf73ef57a2796840467d871dd1126493528568bf6b577c2cbd568c2f109e27c8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":9,"runs_affected":4,"failed_runs":4,"run_impact_percent":66.66666666666667,"anchor_occurrences":9,"after_last_push_count":53,"trend_counts":[0,18,8,0,9,7,9],"trend_range":"2026-04-01..2026-04-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-09","2026-03-16","2026-03-30"],"prior_runs_affected":32,"prior_last_seen_at":"2026-04-05T18:04:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:44]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:44]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:45]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:45]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:51]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:51]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","occurred_at":"2026-04-07T04:04:40Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":6,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":6,"after_last_push_count":10,"trend_counts":[0,0,0,0,0,0,6],"trend_range":"2026-04-01..2026-04-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-16"],"prior_runs_affected":5,"prior_last_seen_at":"2026-03-19T06:26:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000958690\u003e: \n    failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-pmp787\" in resourcegroup=\"rg-np-version-upgrade-pmp787-kwgkwl\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-pmp787\\\" in resourcegroup=\\\"rg-np-version-upgrade-pmp787-kwgkwl\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001248460\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl\",\n            },\n            \u003c*fmt.wrapError | 0xc001104340\u003e{\n                msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-pmp787\\\" in resourcegroup=\\\"rg-np-version-upgrade-pmp787-kwgkwl\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000958690\u003e: \n    failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-pmp787\" in resourcegroup=\"rg-np-version-upgrade-pmp787-kwgkwl\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-pmp787\\\" in resourcegroup=\\\"rg-np-version-upgrade-pmp787-kwgkwl\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001248460\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl\",\n            },\n            \u003c*fmt.wrapError | 0xc001104340\u003e{\n                msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-pmp787\\\" in resourcegroup=\\\"rg-np-version-upgrade-pmp787-kwgkwl\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012d86f0\u003e: \n    failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-b8kv4h\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001247440\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h\",\n            },\n            \u003c*fmt.wrapError | 0xc0005187e0\u003e{\n                msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-b8kv4h\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012d86f0\u003e: \n    failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-b8kv4h\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001247440\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h\",\n            },\n            \u003c*fmt.wrapError | 0xc0005187e0\u003e{\n                msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-b8kv4h\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001604390\u003e: \n    failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-nszsx5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-nszsx5\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-nszsx5, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-nszsx5\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009757d0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-nszsx5\",\n            },\n            \u003c*fmt.wrapError | 0xc000e161a0\u003e{\n                msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-nszsx5\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001604390\u003e: \n    failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-nszsx5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-nszsx5\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-nszsx5, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-nszsx5\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009757d0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-nszsx5\",\n            },\n            \u003c*fmt.wrapError | 0xc000e161a0\u003e{\n                msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-nszsx5\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","occurred_at":"2026-04-07T04:05:16Z","signature_id":"f15184542c39bb774c3be8e478ae04eeb6b9843c71ff6dda60d4522220a27a0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"325ed533ecfab9ad14356e1c3682e386e21134251bde23aff995a4190cec9728","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"38bb290cdcd5fb0fd6fc57972bfe6e55acd2c90b588687aa9b866e4af14f4e0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"4ff69f402a35f8589728a46de86bc483f764602ca78dd11c5bdc70a78155a06e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"58723eeca32480c2a7147989b14ad5770f17d9613a970f9c8d123ee9128db707","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"7ed989922bc03f5a77baa84831fdc451a05ac58926599595e31930fd3a9d61de","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout","failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":83.33333333333333,"anchor_occurrences":5,"after_last_push_count":13,"trend_counts":[0,2,2,0,1,1,5],"trend_range":"2026-04-01..2026-04-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-23","2026-03-30"],"prior_runs_affected":13,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":5}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-07T06:08:14Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-07T06:07:36Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-07T06:07:00Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","occurred_at":"2026-04-07T04:05:16Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","occurred_at":"2026-04-07T04:04:40Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308037523705856","occurred_at":"2026-04-07T00:12:16Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":19,"trend_counts":[0,7,7,0,3,1,1],"trend_range":"2026-04-01..2026-04-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-30"],"prior_runs_affected":9,"prior_last_seen_at":"2026-04-05T20:06:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f2c9e0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ztlcvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ztlcvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00178b8c0\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"9df649c9-418f-4940-bec8-8c0e548e3db3\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Expires\": [\"-1\"],\n                    \"Date\": [\n                        \"Tue, 07 Apr 2026 05:20:56 GMT\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"b0981fa0-473c-423b-8940-929e8b4ed066\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"1246\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"WESTEUROPE:20260407T052056Z:9df649c9-418f-4940-bec8-8c0e548e3db3\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/westeurope/95c613cd-bdd4-496d-928d-a3fc6f2b736e\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 9189BB39768F4C5AA7879357F88C2139 Ref B: CHI261140501031 Ref C: 2026-04-07T05:20:55Z\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000f2c920\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f2c9e0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ztlcvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ztlcvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00178b8c0\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"9df649c9-418f-4940-bec8-8c0e548e3db3\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Expires\": [\"-1\"],\n                    \"Date\": [\n                        \"Tue, 07 Apr 2026 05:20:56 GMT\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"b0981fa0-473c-423b-8940-929e8b4ed066\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"1246\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"WESTEUROPE:20260407T052056Z:9df649c9-418f-4940-bec8-8c0e548e3db3\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/westeurope/95c613cd-bdd4-496d-928d-a3fc6f2b736e\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 9189BB39768F4C5AA7879357F88C2139 Ref B: CHI261140501031 Ref C: 2026-04-07T05:20:55Z\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000f2c920\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","occurred_at":"2026-04-07T04:04:40Z","signature_id":"46a267eec014b8e48c35e8603c9ead475405836ebd27b349d3b51b246bd00195","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}