{"meta":{"start_date":"2026-08-03","end_date":"2026-08-09","start_at":"2026-08-03T00:00:00Z","end_at":"2026-08-10T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T01:28:45Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":27,"failed_runs":19,"raw_occurrences":75,"matched_occurrences":74,"runs_affected":18},"rows":[{"environment":"stg","failure_pattern_id":"aacc0716766c8928f6c0020609ecd8d61148e745e89d627762709d38251c4aea","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-deletion-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":44,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.7037037037037037,"anchor_occurrences":44,"after_last_push_count":44,"trend_counts":[0,0,44,0,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-21-mjzw426l4nbl/cluster-candidate-4-21-vrmnw7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000893d80\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-vrmnw7: failed starting cluster creation \"cluster-candidate-4-21-vrmnw7\" in resourcegroup=\"rg-candidate-4-21-mjzw426l4nbl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-mjzw426l4nbl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-vrmnw7\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-21-mjzw426l4nbl/cluster-candidate-4-21-vrmnw7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000893d80\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-vrmnw7: failed starting cluster creation \"cluster-candidate-4-21-vrmnw7\" in resourcegroup=\"rg-candidate-4-21-mjzw426l4nbl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-mjzw426l4nbl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-vrmnw7\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-9g98w5\" with version 4.21.22 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea5c60\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-9g98w5: failed starting cluster creation \"cluster-zstream-4-21-9g98w5\" in resourcegroup=\"rg-zstream-upgrade-4-21-jhq5xrsc2c5j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-jhq5xrsc2c5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9g98w5\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-9g98w5\" with version 4.21.22 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea5c60\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-9g98w5: failed starting cluster creation \"cluster-zstream-4-21-9g98w5\" in resourcegroup=\"rg-zstream-upgrade-4-21-jhq5xrsc2c5j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-jhq5xrsc2c5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9g98w5\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:741]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000543980\u003e: \n    failed to create HCP cluster np-dg-minor-vtnjvx: failed starting cluster creation \"np-dg-minor-vtnjvx\" in resourcegroup=\"rg-np-dg-minor-vtnjvx-dmk84bgmvg7f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-vtnjvx-dmk84bgmvg7f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-vtnjvx\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:741]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000543980\u003e: \n    failed to create HCP cluster np-dg-minor-vtnjvx: failed starting cluster creation \"np-dg-minor-vtnjvx\" in resourcegroup=\"rg-np-dg-minor-vtnjvx-dmk84bgmvg7f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-vtnjvx-dmk84bgmvg7f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-vtnjvx\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"005aadbb2b3cbd970c8a5647b141355ae07a35e26f7186616adaebc426bf4da2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"019891b44f64af820ab67d4a03e22e90484bcacfd2837d1b3880c34a1a62a2c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"0389f0fe5037dd9b66b12e30917751cf320d942dd5137ab8fa425852b6236b3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"04b06d68f7591d8340cd2cd0a3664e34f6c85b6ea43a98759a95eebfd4c9e013","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"06093355f2ca138fd004f547026862107b9f0a52eb32259575685d2233865485","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"205b77ad65fd814005e59b07bb0f80092412ffb7e55625d71a0c069ef899af12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"2b5581c166102d3e362802a2b9b7ca5db7d9c52ce0179b7a01b7c5f90430856f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"395aa40e0d87b127c9b5c3dce39dda738b17df0a0e193a5b979bb7b7164d854a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"3ed77695497145bee15d3e85c5c26b57a0483536ee6288eac3d3e2a1aa220120","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"3ef03c5aa06933d083c05d97b1201adb1df842a4602827ea97f0c0c70c319e39","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"407f9674f70da743cb87faf636a607306a369ad3c9ec77bd39c7d94b13fbd75a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"47b53df7d22efe440cb378f5f3a4c49f0f4101811d912bc634f3c9e84935ef6e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"47dd5b1a0e75ce0956710d688e8e0d79950d89d69d28307b745d6a2b0407766b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"48239e54f1c1879efe2d363b376ed33c45ba51d725c1c71b25b6ade83b1a8c91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"4dfe1d4ae5dac3d25b96d12833608d17d337c184a3440d8a103a51a7eddaf284","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"580e804d4ca58cdc4eaa5259f692f4f14282a792347cec76bdc3ffe482819c2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"63d9d0524f397d913a5b5a1272dde19792ca755e96ad4927bd730013749b2419","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"752a585057740795b59da33748be6b863be205d712f7d44a1f46ca9f918789aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"7b1079b127846fc162121edef3cfece1fa8d465915c151ef9a643bd53783d55c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"7b5627c44fd5410e19dd20823942c517b968c45d64b70c78e80026f7a0765902","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"7bc7bdb646067d259adb552bcbb44fcd2943e8fa8b45869eaf0db44f85c6104c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"854c876e63dd9b3288dc2c1e25c1701343143d5bad657aa919e71e72638bc74f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"86612854dc9368e3e2c64843502c63b80e2fb6c4566bad1abcfeecca8c28690c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"88c10ad1f25d53953c5a71080d49c8fced5b057003e6e43b188fb5e8f6c786f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"88ee107509854c9048493ea8374ebc643333957d0a4fd8a46b3382181b4ab07e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"8a27f05c27aee4f19310624fba5d87838e218ebaeebba9305a22fe3c7f605551","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"8c93bf65ba2948ee0e2d005a839ba49bf1af635cb9b86e570b44f35681fccfdd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"9278f7958d0b1a68ee9afd3aa124319762aeb67c63316279d91f378e261399f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"98cb12e5c2ca2fdcf044bc271afd3aa65fc347b6a5ae0ec9ce1be1f11a76b0eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"9a6d7a7f74d1e20a435197e5fab8ea6e01bcba87f2ce96006f2ee2612cf51dac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"9b3fff5735545d2eb852868d3b070ee190b2ffd9d65e34224de49792c12e9a87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"9c196c142c882b8a3e2c43b74f6191cbd962560d5ab8633276c85fb26300c51e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"a67308fc6c84a83da2f07d26ab092b47e863c1742537fd0b0b239729a1a4fc0e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"abec481e2d11c454a7ff3f485381e416c410fa9f9bc51b67d7386282b96922da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"ae1e7b4f9f1f94e924c1b536b678d33286c64e9553dbe6fea87c5a787d41ffd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"b2e4d4e8be115b68ea5e116784230ff0a0ded14ad546653f749c2e84e285f4ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"b522cbad0a67ecfef1bc65955eb01c6218fd75ff62b6d93d737fa0b04d44627a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"be7dd2a4109d2586638eb8c6d8840c806cdb4071d4a984b16d0ac9988bf76441","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"c42a1de4f04a03f4642e62e9248d967553be4375b8df079a3400345b432b44c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"c5aff587c361857ab997f807845295ba97c63ace7a5a71be480dfc100b84df03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"ce248cd7f9966c7a758eda1a8b0b2b6938bb0d5cf42a31d78bb95ba908668b43","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"d95d2705cbeb0982d62c28ac72cdf38f2619b0621d7b1b9ea7a35ea40a904cbc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"f3ab84d43b9f2cf3e90787fd823d5c2a6baa7a4093d65468b5640ed6e6343b18","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2084459310561103872","occurred_at":"2026-08-04T02:00:01Z","signature_id":"f5e11620b705af3ffa21e82b238aa2b5c5a12c6658123403cf3046cc0f23849d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-4-23-gw7dgj, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-gw7dgj in resource group rg-nightly-4-23-kg69wjfvskzz, error: failed waiting for cluster=\"cluster-nightly-4-23-gw7dgj\" in resourcegroup=\"rg-nightly-4-23-kg69wjfvskzz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-gw7dgj in resource group rg-nightly-4-23-kg69wjfvskzz, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":9,"runs_affected":7,"failed_runs":7,"run_impact_percent":25.925925925925927,"anchor_occurrences":9,"after_last_push_count":9,"trend_counts":[0,1,1,1,1,2,2,1],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":25,"prior_last_seen_at":"2026-07-23T14:34:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a3c420\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m5xt5hdxpwrm, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-m5xt5hdxpwrm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m5xt5hdxpwrm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a3c420\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m5xt5hdxpwrm, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-m5xt5hdxpwrm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m5xt5hdxpwrm, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-nightly-4-23-kg69wjfvskzz/cluster-nightly-4-23-gw7dgj should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a98cc0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-gw7dgj, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-gw7dgj in resource group rg-nightly-4-23-kg69wjfvskzz, error: failed waiting for cluster=\"cluster-nightly-4-23-gw7dgj\" in resourcegroup=\"rg-nightly-4-23-kg69wjfvskzz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-gw7dgj in resource group rg-nightly-4-23-kg69wjfvskzz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-nightly-4-23-kg69wjfvskzz/cluster-nightly-4-23-gw7dgj should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a98cc0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-gw7dgj, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-gw7dgj in resource group rg-nightly-4-23-kg69wjfvskzz, error: failed waiting for cluster=\"cluster-nightly-4-23-gw7dgj\" in resourcegroup=\"rg-nightly-4-23-kg69wjfvskzz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-gw7dgj in resource group rg-nightly-4-23-kg69wjfvskzz, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a6e390\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-k2sxq2fn67f9, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-k2sxq2fn67f9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-k2sxq2fn67f9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a6e390\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-k2sxq2fn67f9, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-k2sxq2fn67f9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-k2sxq2fn67f9, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2086588427150561280","occurred_at":"2026-08-09T23:00:22Z","signature_id":"ab5d1747b01cb4ed85c52aec4da36e544a51a20c5f33b404314564f5c9fa5ee0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2086226151478398976","occurred_at":"2026-08-08T23:00:49Z","signature_id":"2e35d0637f2265438d03a0d3a210bfffa17c83cc87e4a6e0aa4f6a440c3321ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2086226151478398976","occurred_at":"2026-08-08T23:00:49Z","signature_id":"a393fb7a02dedebda24c1ed178eca7b49f3cb4c483a50ac17af5c0a6109a5d80","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2085863759322025984","occurred_at":"2026-08-07T23:00:48Z","signature_id":"0b041d32ee0f80c34b56423f4816c16db5bbd03c19022810f468a17088f15035","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2085863759322025984","occurred_at":"2026-08-07T23:00:48Z","signature_id":"645b19c134c85f98e801cd2d234be0046818b3b31f786b4d32d4d5752f6aca76","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2085501204577652736","occurred_at":"2026-08-06T23:00:07Z","signature_id":"ac93c66a09aac07d69a5da16c53e722cbead1b5cf33aa049ba9a0baaccd0608c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2085138799490240512","occurred_at":"2026-08-05T23:00:03Z","signature_id":"3ff69bdb204b77ddce696cf3dc401f26fd4f2dd0871fca0f29eeb3708a4955e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2084776403047288832","occurred_at":"2026-08-04T23:00:02Z","signature_id":"cf3274dbcf4acd783be951b12a2d718df527b6813cc74180a8acc05732613d0a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2084414011968851968","occurred_at":"2026-08-03T23:00:01Z","signature_id":"4cb263d1e9a325a7f5898f5a2322e75c383c596b518471172ffb2725b6cd374c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b936783c2d0b2a9d4ffabaeda14053cc9fef881b92be87e0e230002db2affbaf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":14.814814814814815,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,1,1,0,0,1,1],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001694120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5c474lhmgh82\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a693f9a6-2804-406e-bcca-bfe1da58201e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a693f9a6-2804-406e-bcca-bfe1da58201e\",\n      \"name\": \"a693f9a6-2804-406e-bcca-bfe1da58201e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-09T02:45:33.200222612Z\",\n      \"endTime\": \"2026-08-09T03:09:35.968829341Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s2phgsnj590ikiibrfblrlk6f6c4ap3 still exists (deletion dispatched at 2026-08-09T02:45:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001694120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5c474lhmgh82\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a693f9a6-2804-406e-bcca-bfe1da58201e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a693f9a6-2804-406e-bcca-bfe1da58201e\",\n      \"name\": \"a693f9a6-2804-406e-bcca-bfe1da58201e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-09T02:45:33.200222612Z\",\n      \"endTime\": \"2026-08-09T03:09:35.968829341Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s2phgsnj590ikiibrfblrlk6f6c4ap3 still exists (deletion dispatched at 2026-08-09T02:45:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013e8240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-6s2k8l\" in resourcegroup=\"admin-credential-lifecycle-test-zpfkg9jv7t8t\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32887831-776a-4d98-8e11-55767d72496f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32887831-776a-4d98-8e11-55767d72496f\",\n      \"name\": \"32887831-776a-4d98-8e11-55767d72496f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-08T02:24:57.913974421Z\",\n      \"endTime\": \"2026-08-08T02:49:14.472942149Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s240ik84eejtd1sbn7e5qqvj1uv7h99 still exists (deletion dispatched at 2026-08-08T02:24:58Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013e8240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-6s2k8l\" in resourcegroup=\"admin-credential-lifecycle-test-zpfkg9jv7t8t\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32887831-776a-4d98-8e11-55767d72496f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32887831-776a-4d98-8e11-55767d72496f\",\n      \"name\": \"32887831-776a-4d98-8e11-55767d72496f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-08T02:24:57.913974421Z\",\n      \"endTime\": \"2026-08-08T02:49:14.472942149Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s240ik84eejtd1sbn7e5qqvj1uv7h99 still exists (deletion dispatched at 2026-08-08T02:24:58Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016623a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-wddjjs\" in resourcegroup=\"admin-credential-lifecycle-test-2cjl4hgjbxzd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66adb307-293e-42a5-9c87-495ac203b3ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66adb307-293e-42a5-9c87-495ac203b3ae\",\n      \"name\": \"66adb307-293e-42a5-9c87-495ac203b3ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T10:00:59.463734847Z\",\n      \"endTime\": \"2026-08-05T10:25:05.241705565Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s0bflive11oegd4h0qd4ogh1aabqgng still exists (deletion dispatched at 2026-08-05T10:01:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016623a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-wddjjs\" in resourcegroup=\"admin-credential-lifecycle-test-2cjl4hgjbxzd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66adb307-293e-42a5-9c87-495ac203b3ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66adb307-293e-42a5-9c87-495ac203b3ae\",\n      \"name\": \"66adb307-293e-42a5-9c87-495ac203b3ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T10:00:59.463734847Z\",\n      \"endTime\": \"2026-08-05T10:25:05.241705565Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s0bflive11oegd4h0qd4ogh1aabqgng still exists (deletion dispatched at 2026-08-05T10:01:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2086271483448922112","occurred_at":"2026-08-09T02:00:57Z","signature_id":"7ccb40945cf06acef5951d9781cdf6a2d7c453c1d39455d65275b51daa1ce6cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2085909058199490560","occurred_at":"2026-08-08T02:00:48Z","signature_id":"6c6d4aea6bb04d62f4d019f1c472100494a0b18cd181e7a36f48ac7e58fe1969","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2084933694467870720","occurred_at":"2026-08-05T09:25:03Z","signature_id":"2d68d1e5f43f9ba1cac467cd2a47e0d78bc22ce12ba2ab596aa3f26c65eb1580","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2084696847875575808","occurred_at":"2026-08-04T17:43:44Z","signature_id":"5532faf04cb9619454f0dcefc820410741a2a0a0c88a1729b315c6d8a20fc41a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":11.11111111111111,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,1,2,0,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-13","2026-07-20","2026-07-27"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-02T12:11:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:137]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dda860\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-v48bhd59nlrn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72d45537-015f-4192-9dff-7d46dc3292b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72d45537-015f-4192-9dff-7d46dc3292b9\",\n      \"name\": \"72d45537-015f-4192-9dff-7d46dc3292b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T18:17:08.75493953Z\",\n      \"endTime\": \"2026-08-04T18:36:10.634625702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:137]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dda860\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-v48bhd59nlrn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72d45537-015f-4192-9dff-7d46dc3292b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72d45537-015f-4192-9dff-7d46dc3292b9\",\n      \"name\": \"72d45537-015f-4192-9dff-7d46dc3292b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T18:17:08.75493953Z\",\n      \"endTime\": \"2026-08-04T18:36:10.634625702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:191]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001f2e048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tqm8zjwmbxvx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1b17109-deff-4598-bd83-3ef741e315b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1b17109-deff-4598-bd83-3ef741e315b0\",\n      \"name\": \"e1b17109-deff-4598-bd83-3ef741e315b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T05:09:46.184125491Z\",\n      \"endTime\": \"2026-08-04T05:29:03.102580357Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:191]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001f2e048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tqm8zjwmbxvx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1b17109-deff-4598-bd83-3ef741e315b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1b17109-deff-4598-bd83-3ef741e315b0\",\n      \"name\": \"e1b17109-deff-4598-bd83-3ef741e315b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T05:09:46.184125491Z\",\n      \"endTime\": \"2026-08-04T05:29:03.102580357Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001248060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-n6mqxqbs7w4c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a2b5ab8-4c39-4c7c-bc3d-5d14620f0783\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a2b5ab8-4c39-4c7c-bc3d-5d14620f0783\",\n      \"name\": \"0a2b5ab8-4c39-4c7c-bc3d-5d14620f0783\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-03T23:17:59.310024619Z\",\n      \"endTime\": \"2026-08-03T23:37:05.475161098Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001248060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-n6mqxqbs7w4c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a2b5ab8-4c39-4c7c-bc3d-5d14620f0783\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a2b5ab8-4c39-4c7c-bc3d-5d14620f0783\",\n      \"name\": \"0a2b5ab8-4c39-4c7c-bc3d-5d14620f0783\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-03T23:17:59.310024619Z\",\n      \"endTime\": \"2026-08-03T23:37:05.475161098Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2084696847875575808","occurred_at":"2026-08-04T17:43:44Z","signature_id":"3fa4b0813f0517f1a84bc2d149c7cb456ce77f0706f166e4bf1e11dc2a7d7045","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2084499120810102784","occurred_at":"2026-08-04T04:38:13Z","signature_id":"9709569cb9a709106b946e65ba15d1f429697adf279f2f31ca4a5cb4adf68211","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2084414011968851968","occurred_at":"2026-08-03T23:00:01Z","signature_id":"c5d500d4aa4a95776bdbca6e46e169f4726cb56c664c0ef551ea2caf3d6eec90","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d58c4430f859260715a2df79338de096b1f7324f15367436040e29b8a0d491e6","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":11.11111111111111,"anchor_occurrences":3,"after_last_push_count":5,"trend_counts":[0,0,1,0,2,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-20"],"prior_runs_affected":5,"prior_last_seen_at":"2026-07-25T23:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00096b020\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-pc62tg528mmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5afe8e4b-a8d3-4f8f-8545-27aa32d736fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5afe8e4b-a8d3-4f8f-8545-27aa32d736fd\",\n      \"name\": \"5afe8e4b-a8d3-4f8f-8545-27aa32d736fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T10:42:23.203457428Z\",\n      \"endTime\": \"2026-08-06T11:01:35.314522391Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00096b020\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-pc62tg528mmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5afe8e4b-a8d3-4f8f-8545-27aa32d736fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5afe8e4b-a8d3-4f8f-8545-27aa32d736fd\",\n      \"name\": \"5afe8e4b-a8d3-4f8f-8545-27aa32d736fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T10:42:23.203457428Z\",\n      \"endTime\": \"2026-08-06T11:01:35.314522391Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b6620\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-bbncssjxz689\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c31068af-692a-49df-8eeb-d5b62d220f06\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c31068af-692a-49df-8eeb-d5b62d220f06\",\n      \"name\": \"c31068af-692a-49df-8eeb-d5b62d220f06\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T00:37:19.502153509Z\",\n      \"endTime\": \"2026-08-06T00:56:19.182719758Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b6620\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-bbncssjxz689\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c31068af-692a-49df-8eeb-d5b62d220f06\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c31068af-692a-49df-8eeb-d5b62d220f06\",\n      \"name\": \"c31068af-692a-49df-8eeb-d5b62d220f06\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T00:37:19.502153509Z\",\n      \"endTime\": \"2026-08-06T00:56:19.182719758Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014d11c0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-q8r96dg25gkl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4018f813-07d6-479d-8c54-378c34ed46c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4018f813-07d6-479d-8c54-378c34ed46c3\",\n      \"name\": \"4018f813-07d6-479d-8c54-378c34ed46c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T15:09:35.150840021Z\",\n      \"endTime\": \"2026-08-04T15:28:52.741249494Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014d11c0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-q8r96dg25gkl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4018f813-07d6-479d-8c54-378c34ed46c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4018f813-07d6-479d-8c54-378c34ed46c3\",\n      \"name\": \"4018f813-07d6-479d-8c54-378c34ed46c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T15:09:35.150840021Z\",\n      \"endTime\": \"2026-08-04T15:28:52.741249494Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2085314564852813824","occurred_at":"2026-08-06T10:38:30Z","signature_id":"e7357d444f5c28438c07f9a177df2fd3fc74a55f8e855e133ea5fdd687a4343e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2085155530522759168","occurred_at":"2026-08-06T00:06:33Z","signature_id":"86a197ee71b003a39100cab3fda0608a14e1d7ca691062b5b019ad24387dc0b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2084652595414044672","occurred_at":"2026-08-04T14:48:04Z","signature_id":"81f5dbe4532fe7d24ff049e7f41a549dc439944dcb561b6392050ca42f716365","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.407407407407407,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[2,0,0,0,1,1,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-27"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-02T12:11:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015125e0\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nrflz9tns5r5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d90471f4-7f1b-45ea-8ab2-c589a23e2929\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d90471f4-7f1b-45ea-8ab2-c589a23e2929\",\n      \"name\": \"d90471f4-7f1b-45ea-8ab2-c589a23e2929\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-07T02:02:47.753969009Z\",\n      \"endTime\": \"2026-08-07T02:21:54.799592466Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015125e0\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nrflz9tns5r5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d90471f4-7f1b-45ea-8ab2-c589a23e2929\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d90471f4-7f1b-45ea-8ab2-c589a23e2929\",\n      \"name\": \"d90471f4-7f1b-45ea-8ab2-c589a23e2929\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-07T02:02:47.753969009Z\",\n      \"endTime\": \"2026-08-07T02:21:54.799592466Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0014cc780\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb737a48-700a-4f14-af7f-6ea216b75185\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb737a48-700a-4f14-af7f-6ea216b75185\",\n      \"name\": \"eb737a48-700a-4f14-af7f-6ea216b75185\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T23:03:17.301696272Z\",\n      \"endTime\": \"2026-08-06T23:22:17.410372628Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0014cc780\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb737a48-700a-4f14-af7f-6ea216b75185\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb737a48-700a-4f14-af7f-6ea216b75185\",\n      \"name\": \"eb737a48-700a-4f14-af7f-6ea216b75185\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T23:03:17.301696272Z\",\n      \"endTime\": \"2026-08-06T23:22:17.410372628Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2085546498791051264","occurred_at":"2026-08-07T02:00:07Z","signature_id":"9f3b2013df902eddf53eb748a28c528f0f14fdca3b587a2a27a8d1e35acdfd41","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2085501204577652736","occurred_at":"2026-08-06T23:00:07Z","signature_id":"8ba529244f66005297906db010930be8c06ce81d569c22fefdfdd33c456af15d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"212004d426c44c3cfe211cdc6b196a47c64651be7a1c038c75fec2f3c26a9d72","failure_pattern":"ERROR CODE: NoRegisteredProviderFound; detail message No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: NoRegisteredProviderFound","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.407407407407407,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,1,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:120]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dc1ea0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-7pskgt8pflv5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3424990b-503e-457b-a569-03e30b9f05a3\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:120]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dc1ea0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-7pskgt8pflv5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3424990b-503e-457b-a569-03e30b9f05a3\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:120]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00065c860\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rrcfxkncvjb8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/975f1eae-0d49-43a1-a52c-3feacb8cd683\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:120]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00065c860\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rrcfxkncvjb8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/975f1eae-0d49-43a1-a52c-3feacb8cd683\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2085863759322025984","occurred_at":"2026-08-07T23:00:48Z","signature_id":"f9297ad0665bd388b020488542156c55d76b667f3fc6b685771d0ad6bf2a59f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2085501204577652736","occurred_at":"2026-08-06T23:00:07Z","signature_id":"de55e96c4daf1fc979c76963b23c8da6facb26240d60ca3e57de084fffdc876a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4e83fe1c3bb6daff1be6935129ccd1a1c67e021531f7a464980385c362ccc372","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-tc94pz2k8tgr\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-d9x6dw\" in resourcegroup=\"admin-credential-lifecycle-test-tc94pz2k8tgr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-d9x6dw in resource group admin-credential-lifecycle-test-tc94pz2k8tgr, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-d9x6dw\" in resourcegroup=\"admin-credential-lifecycle-test-tc94pz2k8tgr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-d9x6dw in resource group admin-credential-lifecycle-test-tc94pz2k8tgr, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.7037037037037037,"anchor_occurrences":2,"after_last_push_count":10,"trend_counts":[1,2,0,0,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20","2026-07-27"],"prior_runs_affected":11,"prior_last_seen_at":"2026-08-02T02:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00166e138\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"arm64-vm-cluster-lz47frv6c784\", caused by: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-lz47frv6c784\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-lz47frv6c784, error: context deadline exceeded, error: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-lz47frv6c784\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-lz47frv6c784, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00166e138\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"arm64-vm-cluster-lz47frv6c784\", caused by: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-lz47frv6c784\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-lz47frv6c784, error: context deadline exceeded, error: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-lz47frv6c784\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-lz47frv6c784, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011844b0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-tc94pz2k8tgr\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-d9x6dw\" in resourcegroup=\"admin-credential-lifecycle-test-tc94pz2k8tgr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-d9x6dw in resource group admin-credential-lifecycle-test-tc94pz2k8tgr, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-d9x6dw\" in resourcegroup=\"admin-credential-lifecycle-test-tc94pz2k8tgr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-d9x6dw in resource group admin-credential-lifecycle-test-tc94pz2k8tgr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011844b0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-tc94pz2k8tgr\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-d9x6dw\" in resourcegroup=\"admin-credential-lifecycle-test-tc94pz2k8tgr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-d9x6dw in resource group admin-credential-lifecycle-test-tc94pz2k8tgr, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-d9x6dw\" in resourcegroup=\"admin-credential-lifecycle-test-tc94pz2k8tgr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-d9x6dw in resource group admin-credential-lifecycle-test-tc94pz2k8tgr, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2084279024871280640","occurred_at":"2026-08-03T14:03:21Z","signature_id":"b4cd8b644ef7e3e7a265e1a85979f7a338211730fcea7258dcc3c16c24c9070e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2084279024871280640","occurred_at":"2026-08-03T14:03:21Z","signature_id":"dc8c056d2b3b0e56941585cfa72232d1031d9211e7f73e72d91bfe27f8fcc2d7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"655c43af8577be5d2f6f3a0b62e2bfec62cd7a458a7b99b69cb233818f32b1a8","failure_pattern":"API version 2026-06-30-preview should be available for Microsoft.RedHatOpenShift/hcpOpenShiftClusters by \u003ctimestamp\u003e","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:74]: API version 2026-06-30-preview should be available for Microsoft.RedHatOpenShift/hcpOpenShiftClusters by 2026-08-07T00:00:00Z","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.7037037037037037,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:74]: API version 2026-06-30-preview should be available for Microsoft.RedHatOpenShift/hcpOpenShiftClusters by 2026-08-07T00:00:00Z\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:74]: API version 2026-06-30-preview should be available for Microsoft.RedHatOpenShift/hcpOpenShiftClusters by 2026-08-07T00:00:00Z"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2085546498791051264","occurred_at":"2026-08-07T02:00:07Z","signature_id":"fb953757d9381bb4930b52cf81938e4803a7e0941cfd4181b8b5495a5f2e5140","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3b73b158de2cfa00931d006d7dea813139c9ea99ca292b8ded3d10d4beb4674f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.7037037037037037,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-7q4c88jh8t9s/cluster-candidate-4-22-lmwnq5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a92b00\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-lmwnq5: failed waiting for cluster=\"cluster-candidate-4-22-lmwnq5\" in resourcegroup=\"rg-candidate-4-22-7q4c88jh8t9s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0fbf75bb-5e9d-4978-9631-825e28be408a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0fbf75bb-5e9d-4978-9631-825e28be408a\",\n      \"name\": \"0fbf75bb-5e9d-4978-9631-825e28be408a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T09:46:42.016692331Z\",\n      \"endTime\": \"2026-08-05T10:05:54.175998153Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-7q4c88jh8t9s/cluster-candidate-4-22-lmwnq5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a92b00\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-lmwnq5: failed waiting for cluster=\"cluster-candidate-4-22-lmwnq5\" in resourcegroup=\"rg-candidate-4-22-7q4c88jh8t9s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0fbf75bb-5e9d-4978-9631-825e28be408a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0fbf75bb-5e9d-4978-9631-825e28be408a\",\n      \"name\": \"0fbf75bb-5e9d-4978-9631-825e28be408a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T09:46:42.016692331Z\",\n      \"endTime\": \"2026-08-05T10:05:54.175998153Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2084933694467870720","occurred_at":"2026-08-05T09:25:03Z","signature_id":"539110717216b0bd4afd3ed94982bbacf11388db28c55a00020571c7a52cb506","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.7037037037037037,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fd8420\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-p7npg92ddx4m\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a0729b30-0b22-49e3-805f-acf532cd3054\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a0729b30-0b22-49e3-805f-acf532cd3054\",\n      \"name\": \"a0729b30-0b22-49e3-805f-acf532cd3054\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T23:26:29.027786333Z\",\n      \"endTime\": \"2026-08-05T23:50:29.922451032Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s0n6iovcvlkt9agtmggl0qa3l8afqkg still exists (deletion dispatched at 2026-08-05T23:26:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fd8420\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-p7npg92ddx4m\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a0729b30-0b22-49e3-805f-acf532cd3054\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a0729b30-0b22-49e3-805f-acf532cd3054\",\n      \"name\": \"a0729b30-0b22-49e3-805f-acf532cd3054\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T23:26:29.027786333Z\",\n      \"endTime\": \"2026-08-05T23:50:29.922451032Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s0n6iovcvlkt9agtmggl0qa3l8afqkg still exists (deletion dispatched at 2026-08-05T23:26:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2085138799490240512","occurred_at":"2026-08-05T23:00:03Z","signature_id":"a8b964b2f40b39ef0419e88b4d1a7316651a17f393adbe43bf3330cf60f5c4fe","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.7037037037037037,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f92ac8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hfkh8p5xrvlw\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41936837-e6d6-44be-998c-7d15b68e8682\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41936837-e6d6-44be-998c-7d15b68e8682\",\n      \"name\": \"41936837-e6d6-44be-998c-7d15b68e8682\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T23:24:00.311269145Z\",\n      \"endTime\": \"2026-08-06T23:48:08.178392096Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s1c9mdjf1ocuvc28u4dhg8pvcm8nrjo still exists (deletion dispatched at 2026-08-06T23:24:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f92ac8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hfkh8p5xrvlw\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41936837-e6d6-44be-998c-7d15b68e8682\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41936837-e6d6-44be-998c-7d15b68e8682\",\n      \"name\": \"41936837-e6d6-44be-998c-7d15b68e8682\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T23:24:00.311269145Z\",\n      \"endTime\": \"2026-08-06T23:48:08.178392096Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s1c9mdjf1ocuvc28u4dhg8pvcm8nrjo still exists (deletion dispatched at 2026-08-06T23:24:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2085501204577652736","occurred_at":"2026-08-06T23:00:07Z","signature_id":"bff4b9d0a55a12d86c3053c23e58d2876a14243246abf5aa351510db9635ae3a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"237c6decda5c048a5babbacbfa611ed464ee088b0239982f9244e964943612d4","failure_pattern":"v20260630preview API still not deployed as of \u003ctimestamp\u003e deadline","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:91]: v20260630preview API still not deployed as of 2026-08-07T00:00:00Z deadline","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.7037037037037037,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:91]: v20260630preview API still not deployed as of 2026-08-07T00:00:00Z deadline\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:91]: v20260630preview API still not deployed as of 2026-08-07T00:00:00Z deadline"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2085546498791051264","occurred_at":"2026-08-07T02:00:07Z","signature_id":"7269898a1029a630ccb580ccffd8c4ae1c6eb7659de2a27a8754827626d9330c","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}