{"meta":{"start_date":"2026-08-03","end_date":"2026-08-09","start_at":"2026-08-03T00:00:00Z","end_at":"2026-08-10T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T00:46:37Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":69,"failed_runs":27,"raw_occurrences":249,"matched_occurrences":249,"runs_affected":27},"rows":[{"environment":"prod","failure_pattern_id":"4035e44f5e2eda00878b0b1b6c1eadc2bdf56ef52966cc244b842b956c95ea0d","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-deletion-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":127,"runs_affected":3,"failed_runs":3,"run_impact_percent":4.3478260869565215,"anchor_occurrences":127,"after_last_push_count":127,"trend_counts":[0,0,83,44,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be9f40\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-zgrvhfc5n22r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-zgrvhfc5n22r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be9f40\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-zgrvhfc5n22r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-zgrvhfc5n22r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-20-sq4lngxskx9k/cluster-candidate-4-20-ktfxxt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e76960\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-ktfxxt: failed starting cluster creation \"cluster-candidate-4-20-ktfxxt\" in resourcegroup=\"rg-candidate-4-20-sq4lngxskx9k\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-sq4lngxskx9k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-ktfxxt\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-20-sq4lngxskx9k/cluster-candidate-4-20-ktfxxt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e76960\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-ktfxxt: failed starting cluster creation \"cluster-candidate-4-20-ktfxxt\" in resourcegroup=\"rg-candidate-4-20-sq4lngxskx9k\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-sq4lngxskx9k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-ktfxxt\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-21-f69fr7gsvs5v/cluster-candidate-4-21-7bfkvq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b83160\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-7bfkvq: failed starting cluster creation \"cluster-candidate-4-21-7bfkvq\" in resourcegroup=\"rg-candidate-4-21-f69fr7gsvs5v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-f69fr7gsvs5v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-7bfkvq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-21-f69fr7gsvs5v/cluster-candidate-4-21-7bfkvq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b83160\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-7bfkvq: failed starting cluster creation \"cluster-candidate-4-21-7bfkvq\" in resourcegroup=\"rg-candidate-4-21-f69fr7gsvs5v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-f69fr7gsvs5v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-7bfkvq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"03ca5588a6d5ccfca9529ca352fe15e131dc14fe6b97fc8eaa31b68af0d37fb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"03d9f2660879ea7816bc81e205b8dd4619bb6aea8225c4136453f47f50eca4d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"09de15d9624444e93e6bf987dffc833cd53e39c16a86437e97fd60913d644250","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"10de3f5d12e187c4ba880683b77390baca238e257952271f22d35c2870cb12e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"151bb421df6c35de4d71a921fa0652ef93b9d4b444d74511256c55bf88999c77","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"1c31c6bc3f9173f506da8b1b1183e76cd9b1b75187d50dabe6dc50fd5555aa71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"1e158c248c7895a2968fe0b4467091c42c9b8d3853a0e1d3b43fa15e2008d959","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"23e5f822309cc1e9c01f2db4de7fae3c72742de7f45518582492346bc4edd84b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"2820100c541c9229944937942cb8d787d42dd0871517a5115ff54a877d664822","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"318ea146134173a78f55b8d68c490958f9709535bde129bca8cba71dbd3b430e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"3b54516e07b66402eb815a7135d72999c0f847f54d60d7268cab74840aaf2e2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"456c3166931597502adeb28ddcddd2fc60b544fea344bd8285f605481c6f19e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"48c508e42520522a03b7c669563534004de1b261e43e421ddcf62557809cc962","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"4abbc1a05acca2f17a270bcda87e4f3edee9ecd69c41d9296dacdbc268ee8b6b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"4eb8cf5247e1175624ebc586f4a9ceadab8907b430637a2022df29df86cea6c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"55172ed61dca08ebe3b1b0ef1b612788bf3981ef588c9b9c7b78d353b2dc8d58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"5b18f47b720381408ae0a10d5aa276cc06ca15f1c1e3615f528f23deaf94c488","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"613639b34c0ba2c7fa43b538fa0a165cb1d21c6c86f7b5482797565d2c8352a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"6cd81fc9f402c323bfb4e4df5ebc0a57b6261e522a6c966e6b69342262e84373","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"722f43dd6402a452b50618297d9730370ac258282b7694ba0b75b9aeef5313e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"7e9066c154a432fa31bf0f54661eb7f706f23595b21dc7038bd24a3a5a5a2bf4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"888e42ea7a2f0fd3efb3efdcc0eed37ffce6bd2fd21808f1e88f8e5279b10eb8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"8c44d492c1a869c6423cd4cf609342a5c1acfe40f18a99411fa912e738a2ac02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"9542edd37b4734b7d505b5d73cd202208c55322a80121ce628b74fe4f974adb1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"9c9f18d5ef1ecdcb9b9cc30cdf1b197e5b0df76bb0938ee6cb8d0d2bfb66df11","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"9cbafb69ec331ca9dacf8b78fd024215f761d05d2023a155eb397ea8dc073424","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"a103841e1f920539b9907f65dd4a97e83c4feb8dc319d414cd89508f45697eb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"a1b7fcdc4526d3eed3bb2cf0e4f93d318cee6f6dfb7b5434b250c5d801f9b97d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"a359271faf30ecc2ff4d41db83d5780b53fd9ef4ea590ae481df44368e4a560c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"a8fec5104a40af90d4881cb81832d5fa132463243203001e06e18a58c952b4a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"b01aa812bb6a54b76c9f2192d7e183da3982cb9e52eadc2bd9490b0b906ae9e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"b0ac9daad8990dc03094a99b65d4ba485d8b6ff900ae5c3600e4293380b80460","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"b21afd9ddd5e9f5de4d6b0bcb4a6187759b9e17f75c2b678c0189819d833a1c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"b2a553e93b5a9584ee36e839f129be259641ae199e7c28b504e27cea02ca55fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"c21d772b22b9fb8ba77e87dbf4c3ea34266aae237b9a4b71622bfa8303b6172d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"c2a8f7be5df54c2520c94a9c655b2dd19b7c187e4bfef26bfb6e226a76d7b146","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"c5f3cfb7ed0681d486275ae9f03df21ec5a0168a6adbcfb1491ca5bfd171d7e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"c8ae398eef451111ba70a990fb1a9b871cd6c4e44a2c96275462d5389951af37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"c9ec6a53cd25e43a397a9d4056dc2aefe0b2eff4d58819b25725d596987dbd8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"cfd01877049d84840a5266149d64e2b9743dc659e4cc3a4a4f535d21d7dabd0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"d07f2da0734e9ddb73b555867ec875d02926f288b0620f9666d9c5461223778b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"ddce7ca7bbc12d124d0cbdd62dd6b20519e10df13bdf13de1e482dce46a207eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"de851742dce32fec2b63776b872ae3b81315b96c525bda9abb50f8f288dc8b71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"f9fff26eeb6b461f3d4bf4c50cdf407684a6ab471fe29a567d90fe82be99b1eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"03a6900216524d3fc5c01b3c9dcce1e3825a6b456b8d770a46df5456b27f38e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"051375b60fd0fcecd39c6e7806d0292b17e0f2f0dc29680702e5328cabc66e55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"0572a487269495833b736e71558c9334bf58351f805185ccc6ceee7ea8c4cc64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"071906a1ae00e9f51259d3b44cfe460d70b9e3d49eb6c15d26475d3735380bde","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"0df271087432f427b0e09e19a3fa04e12a670d0423b2285d72d4e93d3645b67d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"0f02500270f31bafd266d6815349d68f6628d8fea4aa040c50b56dc262c345ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"11597f10beba3e3b909bd33d97a0844aff3f32cdc1bbc1b6d85229bc1ff6655b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"16190dbb45251421c4307d6cb0d09647c83e04795c52adbc4821e7a7781888c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"236312b372f834e0da663c7b9cbd17e90814ece8f44f82a9a02c1d4daa4a6679","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"2b4f10e60519ed7368c1625e435097482f9e2d7ab0612c3e75d65a1eab2d82d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"2be5cd0ecee42a7b189f1ee33d0f5170efac398864a868d68421afd08bfee84e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"314e425e642e40ec34e95f6912a0e4f1f26b5fa0b804b2b77459ad2901fbf35f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"50b4c9df98e492e17fcf889bbf5ce7050dbfdfb7db6813a45d5d1a19641df163","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"53237717c0128630877d1f8e035c2d432e3ce8b97e379b983d11b11f3c33e627","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"5630cda8f7a77ccbf08e7218f1b5f6df4fa5cad309b9ead591181ee53a36f0e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"5adc265944571a6605167bfdd75c191e60efa657c86fa3e7b7e41c40b41edf9b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"5dbd86b9f467702f03c4e1930dea0b871ca81d98278201e4c052bdc707b2450b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"5eacfa9d7c934b75278ff43ede427d7e150dcda5169e26f7315e79f6be3631b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"6cdb2512490e168b4d2ed0716d5884ecfe57f5c0cbf51aadb21c4103c9d5a0b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"737d7dabc009fd4cd53bac6a3b24ff02eddca578c9cbc385b826098e9325de01","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"7b8aa3c3083945ad2ad74bd70a34760e9c71a4ca0f2a7abc8c82c21f56b246d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"821566ff4331bb4c661a9d3daf1faeef695de7e9a24cbe12891bea27d527b839","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"82841e97af2cc9cef11df86e354bd0856d399013e604166c0c1c4645b4ecfd2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"9ee6b2b7608ba00bb94d0e4179bdae1a036aba0bc32b01defdbcb95970e4ed99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"9f980c4e7b0e3692d69d256bc70d316e3d6ef839c3a1b3f330cc7ee44bb173f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"a0eb033199530f71b90f2af1bd5c0087504a449b1491c5d377b2161be8024113","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"a410aac49b64e4a5f9c52ff7aeffd10f87c3932eb5cfa0d20a3321b06c484c96","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"a6b15b552c6dd645f35235ce29a13afcdb6262d5797327a07b01828b1db10650","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"a802c05baaffc7a8fd5a3fe4448021c8485477a656351a457077a9d5b64a9be6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"a8e431b5b8a6975f386b491800761877424bd2c9af7a31fa8fdb6356ecdfe7b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"ace8d7ea09281865d53dd90848a49e8956fc226eb8e7a116e0fb8f3f8d3811c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"ae1dffb2aec4ac3c523139c244f4a9a599151476f476cd40ab5064e272186796","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"b1ec86785100303234a5cd2bf5f375e6fa18e7eec78275d1b5c3ebe9009b8344","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"b1f3140c4c3787fd34aafbbb29ca0bcca0b06e61a0c122a102335ad0ff59b86c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"b6260c1e6d58c4b4726a14963fe6256a36dc61307f012b3badf518447491414e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"b6f8ba4ce164ff901ec6aae43b213b7f95887d4d4abce6df647dbfca74298ac3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"c7f382bf45f18dc6f789be46eb5a386fca07765e71b00e029eb9372455baf2d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"d20077aa494d95559385923be773824d729718a57448ecc828379ac7f2dfc4ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084776403005345792","occurred_at":"2026-08-04T23:00:02Z","signature_id":"e14c0a6db55bcc05716ca586ecc0bfc65df8c9373e193f6224a4bbb1410bd1c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"065caabc4c5cfd9f508b78214ff26fe0913b43d98cc863d7be953ec69d27dcf4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"06e19785aa95db254dbd783192a7542483d7ca582176bbf5da30456cb71a98d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"07602e13870d696af679f66c9f39ddf6ebb3cd0e60ef9ab6e9022550601dbff4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"127eae4b2ac466a215eb52ac2bfed67f2990d76a2319f03b4578681291032bfb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"1827ada2a1f65f3297e8b6bda0c7aae9c4fb4c0037c5c00475c3b16f77e6ac4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"1e4af7315422d6089d9535f75dbbeb47da1cbe19c7b27d149477efe349d3982d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"22875f05ba487cc2b5f2c67ca60d5cc1794cdd7a2f29c73476a4c97870812d40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"32b3b2b25779a4aa5c22cc1fb00d8d37d118515b9af3215ca1c6b8d1ddec230d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"355a15f5dbe8bd24919cb70e0cb7e7e3599438792dc1c4ebd0fdd2ddd6b72d86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"3704a47c13b3aa070ee822bcdc53f4c38d5f293678e415fd08d1e598e7e02679","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"4a475e1db369b510c13498bc5f9fa6dedcd233b974c9a6f379b0456f0f692cbc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"4ef009189d79947285acf379f2f8095c5fb658bb0dbeb02544e26b273cf6f1a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"51dc20c0f8c9a9916c7d1fd4a111919e7836a6d60d616fd4551d9cbc7800722f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"6bae8aeab2debd060e0044351b247536c2a12e26cdff98691420c5effa83980e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"7089fce01bb1d479be2eac3658f3b69fe492edf7ed4e699de632bcb3fff40963","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"740d1b10df00bff5c93465bb8922f0ddaa703965a0cc48a39282826d4f5de3a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"76262ba8e29e4d9fe094b4c3dfb712ffe495116235f62e5a4a65c7a509090c99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"803a74dd2d84c4dc1a14f5bd90952575479c9bac51ecb2c78ff58b5642d07a75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"815eb6616776e879ba22fbb302824737fe2281c6ce4037ab48a4c604f002b236","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"8a865d1a4b5539775c017fb9af34bc5d97c9c4813232f082660d0828819983f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"99cc2d156cd65df232d272b70a19d03337a0252264cff524002bdba6f2f5dbb1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"9a7203ed8ef1bcdbb218d24d03fb524ecb7aeb7ce1287a95e940bc6670aa4c4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"9a7edc59a45b85e2548aed4b1e0475481e83df09d4a49729335383e9f1cf50d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"9c9bf3ec6e4cdd8963c4222b418e06edb45bb26f013cd3cc694b9857f443e949","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"ad2cfb85edfb814e8dac4c0fea5a7256b704d3f4cbddb09c609459de6ce6df1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"aea067320b95d92d08446b1155ad195a80a6c3c00c9a95a5ae2d37a5893d61fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"b3052a4bd8d0338ba0ac4e6cb69ed0e65648582367c86add85f4beca2d75e072","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"b3bc27c2ea4fd41e61ad49eb4171985a80e6db2a0675aa96280dfed9f6c62c19","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"bc8b15ba98f5aba2c5f23f12f261b93acb9196357ae6193bcf0fa876b9340da4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"bda70efa5c40aa1fb731c6d2de753b1f3fe4019d0dd2889d548a6e94295dfcfc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"c6f3a3403629057b7e2d14617f9e8713c8ecb1fa630fbe4270be52b416a67da9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"cbd93a126b5fa4a6ac649845f69f9b02537351bdb17002461c3124b5909f0993","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"cdeaca2899a522611bc5b53a788460b9284906a639e0846f1da6ecf5a0212721","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"ce6cb2dfd7ca2ac65e46900e61e2a8cda887176d16f96a1cb394f856f1a09280","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"d44ba76330eac190abd09b0d61ea7efe6aadc307d72804ed52ae9298aa76a78f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"d7991c7f71e1ee1836ded6fd73ec5ed085cc895852e2e0d43863de3492c07560","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"d9e4b21bc2936f34371a3e7e87fea60b451471d628007a79c4f91d2bc5b4a4eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"e328586d49a079946004d799da6d00ba4da74435fd4129e41a1bbd31b7e61afe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"e53b39349ead9935723bb9ce5fa656796637efe253b57d1d1e6e87cf45a048e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"e6541e71661a6e219aab2303c84f86623afda74315d9f9bba952828b60e46603","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"e7042657c6b1bdd45bece9b00e68c44733c98f007671913bc2e93781dcb0ba4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"e931912b6c276b44b5e91fb255cbd29661f41ae54a6eb41a6696b7bfd6b2a2a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"f2defcb909ede7e35500e856679bf99b6c55df27430a6831256e017e5180edea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084459310535938048","occurred_at":"2026-08-04T02:00:01Z","signature_id":"fee1322c51c7e642edc1daf65a9d188dbec3fa1d6cff0bcf38b3f72dfcbe98da","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":28,"runs_affected":6,"failed_runs":6,"run_impact_percent":8.695652173913043,"anchor_occurrences":28,"after_last_push_count":119,"trend_counts":[29,0,12,12,4,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-27"],"prior_runs_affected":16,"prior_last_seen_at":"2026-08-02T09:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:182]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-zpdmdp\" with install version 4.20.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00083a720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-zpdmdp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-zpdmdp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-nt59ffl77spl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5bb285a7-7ce1-4027-9552-2a08322f2e58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5bb285a7-7ce1-4027-9552-2a08322f2e58\",\n      \"name\": \"5bb285a7-7ce1-4027-9552-2a08322f2e58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:20:54.106332052Z\",\n      \"endTime\": \"2026-08-06T13:40:00.506003922Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:182]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-zpdmdp\" with install version 4.20.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00083a720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-zpdmdp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-zpdmdp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-nt59ffl77spl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5bb285a7-7ce1-4027-9552-2a08322f2e58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5bb285a7-7ce1-4027-9552-2a08322f2e58\",\n      \"name\": \"5bb285a7-7ce1-4027-9552-2a08322f2e58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:20:54.106332052Z\",\n      \"endTime\": \"2026-08-06T13:40:00.506003922Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-qgwpct\" with version 4.22.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001a9580\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-qgwpct: failed waiting for cluster=\"cluster-zstream-4-22-qgwpct\" in resourcegroup=\"rg-zstream-upgrade-4-22-sq8l54wqsbkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bbc9d45-d885-4a5e-a947-9f6cc88c4261\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bbc9d45-d885-4a5e-a947-9f6cc88c4261\",\n      \"name\": \"4bbc9d45-d885-4a5e-a947-9f6cc88c4261\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:19:49.664906309Z\",\n      \"endTime\": \"2026-08-06T13:38:50.306742743Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-qgwpct\" with version 4.22.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001a9580\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-qgwpct: failed waiting for cluster=\"cluster-zstream-4-22-qgwpct\" in resourcegroup=\"rg-zstream-upgrade-4-22-sq8l54wqsbkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bbc9d45-d885-4a5e-a947-9f6cc88c4261\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bbc9d45-d885-4a5e-a947-9f6cc88c4261\",\n      \"name\": \"4bbc9d45-d885-4a5e-a947-9f6cc88c4261\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:19:49.664906309Z\",\n      \"endTime\": \"2026-08-06T13:38:50.306742743Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-ntl57w\" with version 4.21.23 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9cd40\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-ntl57w: failed waiting for cluster=\"cluster-zstream-4-21-ntl57w\" in resourcegroup=\"rg-zstream-upgrade-4-21-tdr6mr4275wq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/00f2afbd-cf95-4467-a26f-0c9ca61e32c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/00f2afbd-cf95-4467-a26f-0c9ca61e32c3\",\n      \"name\": \"00f2afbd-cf95-4467-a26f-0c9ca61e32c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:19:35.127061483Z\",\n      \"endTime\": \"2026-08-06T13:38:40.310199162Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-ntl57w\" with version 4.21.23 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9cd40\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-ntl57w: failed waiting for cluster=\"cluster-zstream-4-21-ntl57w\" in resourcegroup=\"rg-zstream-upgrade-4-21-tdr6mr4275wq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/00f2afbd-cf95-4467-a26f-0c9ca61e32c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/00f2afbd-cf95-4467-a26f-0c9ca61e32c3\",\n      \"name\": \"00f2afbd-cf95-4467-a26f-0c9ca61e32c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:19:35.127061483Z\",\n      \"endTime\": \"2026-08-06T13:38:40.310199162Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"3596543c1f0a076fc250f32a4b79935d6f90a8021f9e78f19a42f1a51a6b044d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"77cfad70a10a8d790241b041e10651a57bd31952e4da039a060f83714fb72f7a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"a668f0b0a6511199b7c6d0a5e0fc0e43d56beafa0b386871867965a9f1ffc737","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"b3f0bdaaeab67fb751adda6e6d84a48a238cd3ce11351f6595273141a8b88bbd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"08598c702ff15f33afec0e5639dc73f776279c2738641d0d10b0824a07a05a29","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"2dd17e0d6d0b4aec1c387011f2065976166e2b25b25e613812b7d9ed8ea630c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"36ec2f0faefdd64ee3141448f5d7ea2bb4f7ca992a71361ad75d894f6ada2eb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"4556ea65211f0beb6cb5f0494fb1adfc7b76928ad5716a56e3ca1c9711370afd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"5461a3f3f95f6df7ae047d1d5621eeebb5562cc076ff4866e9fb27cfe652beec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"609f8f37aef29a3f82de90d62497d16c206e0bc090300c6407688069233fd07a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"89f055d074b543fdbd1f9269584a9f82284e86bd09ed144dbbeeaffcb4c30b3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"9226d2f2aa986e80238477b17e68fbc96e81f25d70bfaa783bbdd2a73cdca981","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"ac1ba843d5790f1712ba1c5f6f9dd81e26b8223bd2b53890d944195cda747975","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"b165651a3935c38e4cad6651ba04430382ce5597407df7ba93b839e61bcb687c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"c2df7b85900c9a7db7d86d3cfa1c61c534f326f91db67f2a1149f4cbd8ef1eac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085136126972006400","occurred_at":"2026-08-05T22:49:27Z","signature_id":"8a2200ed713f2a7af80f6b1e67f08f27206d0da45a7832e6169b2270ce7b206c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084760764756791296","occurred_at":"2026-08-04T21:57:53Z","signature_id":"231e25b154c48b4ea62386522b434146f889a5c8831e60e63338dfcc31595dd2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084760764756791296","occurred_at":"2026-08-04T21:57:53Z","signature_id":"2ea2027e26f15adbc53ff9cd4368180e24f3a9758bc0e23baab6ec9a3791929f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084760764756791296","occurred_at":"2026-08-04T21:57:53Z","signature_id":"9a0864faeabb48d70a30031e5ba63af7646bceab117efe578504d076bf9ca846","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084760764756791296","occurred_at":"2026-08-04T21:57:53Z","signature_id":"a643cac0d1569ab8cc30a82ef426424303b6ed080969e63581a5fa301ec23f6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084760764756791296","occurred_at":"2026-08-04T21:57:53Z","signature_id":"d93ac4aee5f7f5f60af42da0b168e918e3376156b84f4c7a339fef27aa74d2bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084719557146054656","occurred_at":"2026-08-04T19:14:09Z","signature_id":"044d69bc7cee144dd0878c3c6268205decc3623aa7caa5f1ce3369649d997c06","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084719557146054656","occurred_at":"2026-08-04T19:14:09Z","signature_id":"6a84028633655974b8f4eb3bb83e2c9703d8050e161a1919cb8844e8a1813746","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084719557146054656","occurred_at":"2026-08-04T19:14:09Z","signature_id":"9aaa50d47fe5595692afbfd7398319a83fb475152ddc8d71372d48bdd04b8a07","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084719557146054656","occurred_at":"2026-08-04T19:14:09Z","signature_id":"af0f5a63f8a736b77e278800eadd0dd6d7f5fb5575e23cec4c6159f922fdea05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084479621713104896","occurred_at":"2026-08-04T03:20:44Z","signature_id":"3019f19360d1ffff14314c8793e0ddf10e39c84f3ff643507e6386060fd12ce0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084479621713104896","occurred_at":"2026-08-04T03:20:44Z","signature_id":"cb66cd6bffcffd0098f20b5bcc50029c994a6e7ad4915e9dd04a8cbdcb44e6da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084479621713104896","occurred_at":"2026-08-04T03:20:44Z","signature_id":"f5ba219a3340b852969c774ec3e66d9debbc3031d811bae2e9b27431f2b04d3b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"failed to resolve default VM size for node pool np-1 (check VM SKU restrictions/quota for the test subscription in eastus2): selector \"default-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":27,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4492753623188406,"anchor_occurrences":27,"after_last_push_count":27,"trend_counts":[0,27,0,0,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:127]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0029e2940\u003e: \n    failed to resolve default VM size for node pool np-1 (check VM SKU restrictions/quota for the test subscription in eastus2): selector \"default-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:127]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0029e2940\u003e: \n    failed to resolve default VM size for node pool np-1 (check VM SKU restrictions/quota for the test subscription in eastus2): selector \"default-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:174]: failed to create node pool \"np-wcfp26\" for cluster \"cluster-candidate-4-21-wcfp26\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00116db00\u003e: \n    failed to resolve default VM size for node pool np-wcfp26 (check VM SKU restrictions/quota for the test subscription in eastus2): selector \"default-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:174]: failed to create node pool \"np-wcfp26\" for cluster \"cluster-candidate-4-21-wcfp26\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00116db00\u003e: \n    failed to resolve default VM size for node pool np-wcfp26 (check VM SKU restrictions/quota for the test subscription in eastus2): selector \"default-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:155]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002dae080\u003e: \n    failed to resolve default VM size for node pool np-1 (check VM SKU restrictions/quota for the test subscription in eastus2): selector \"default-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:155]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002dae080\u003e: \n    failed to resolve default VM size for node pool np-1 (check VM SKU restrictions/quota for the test subscription in eastus2): selector \"default-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"08ea44a748fe8c21b71f718203a2a57cd6929b84313e544f227b5a0346eda99a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"0bbeff96259bc2fe72babf01696f75c130bdc7e65a9db42eb3208e17bdf30bc0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"10e9b64ec048373fbbdf6e708fd3f6a57b274e1d5ca04bc699183ab3deee920a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"146165c074532934bb3d500d38bf08ff773b8945f7dc27ad8bd82aa3b82bb71d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"17c3b756dbda29a5b27c65cb1b2fd4cd6ff14c5e41280d49784088972d50d4bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"1c2e2578fd253599169459ba8a530215cdb9c08c4defb9a0279155f461c05852","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"2337189771ca46ff9a68466baa5d06f9d8b20a3391f7ce7483c16dca64e2d0ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"2d739b50b3b655058b881f2e3686cec4b810ce76fb448b6218035bc4c0c53ee9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"4e7d10168b3c6c418e107e585c401e6d29d5e53cd774a6ea4d14b176baca7aeb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"6143b4a182a5d674a1e4fcb9943dc1c055c1a5831de11238a380eab541b2f08f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"6796b938bf27f3104c664a261af6712662c52e5ed233ff54b96f69f1f7cbb5e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"7379574235dff7d40019fa9375a937ebd20a84ef19b69e8758a1b78011c57819","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"7f25a3677bcd271448f1fbdbb86cc27d9635dddfd08f0715cf6e5cd4bfbaae13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"845dca57b475f8702c10eee34deb73900c6b167dd0a2bf5e0a70e7f452bb709b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"8e5fb39e0cba5872a428e807a9a881a7830f8b2bc1b189e4bb429eb270153c02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"8ec3736381363d981bb71209a70eb7c9ed2edc403c1bb38cb102a64ad47e6bcb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"971d0bd8b83bc312aedb0d5f9a8c5596dea3164d6ae7fc4902a03781f6f35bfc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"a973a1af7e922d5553ecdd3b0689ca1aaa0e867c99b2d8d77d293bdf3095e371","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"b2c808fcccd4ee7bbe812807b2e1df197106b8813cd05ffa6f1bf7029f0c1b66","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"c2332feb102371440193714336c94a8c5f1c6204903980a871dae802b081dc8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"cd6a4146a8ffb4e22a63ad62670fbb1fa0ed605434a3629acdfb87ff60001b63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"cea9f8bd7a5f67edfa86b4df32ab70ecc8b06a661e5e45682284da6d53404762","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"df8573f1c523a507e71e060bfb8629098968eee0eb287a2784c01e65e235c691","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"e4173ad48a3252b5bd859c301be8f85e8feebb7f691bb1dc05dc634499c44830","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"e66bd336f0f79518a6ea485f880a39898f6a2fa5539a4f744c7e9fffca405c75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"eb4a6a244383d5698a98d6cc8deb3d8b8521c4578111d69cfa620b1de1b260a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084334367747870720","occurred_at":"2026-08-03T17:43:32Z","signature_id":"f3e172c7c3eeea5f3ae95bd7785a4c139e17789c05badfa776ad390935ad7734","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":18,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4492753623188406,"anchor_occurrences":18,"after_last_push_count":33,"trend_counts":[6,0,0,0,18,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-27"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-02T09:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000880f00\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-pnf2fh5wjwql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8925daa4-519a-42f1-a42f-eda89e216058\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8925daa4-519a-42f1-a42f-eda89e216058\",\n      \"name\": \"8925daa4-519a-42f1-a42f-eda89e216058\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:29:35.264600406Z\",\n      \"endTime\": \"2026-08-06T13:48:41.617977129Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000880f00\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-pnf2fh5wjwql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8925daa4-519a-42f1-a42f-eda89e216058\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8925daa4-519a-42f1-a42f-eda89e216058\",\n      \"name\": \"8925daa4-519a-42f1-a42f-eda89e216058\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:29:35.264600406Z\",\n      \"endTime\": \"2026-08-06T13:48:41.617977129Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00165cba0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-b2xd2cfggh75\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1b4f6d3-9523-41a1-b21f-c572ab6386df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1b4f6d3-9523-41a1-b21f-c572ab6386df\",\n      \"name\": \"d1b4f6d3-9523-41a1-b21f-c572ab6386df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T14:06:00.450328897Z\",\n      \"endTime\": \"2026-08-06T14:25:02.805147479Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00165cba0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-b2xd2cfggh75\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1b4f6d3-9523-41a1-b21f-c572ab6386df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1b4f6d3-9523-41a1-b21f-c572ab6386df\",\n      \"name\": \"d1b4f6d3-9523-41a1-b21f-c572ab6386df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T14:06:00.450328897Z\",\n      \"endTime\": \"2026-08-06T14:25:02.805147479Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-wjpjjj with version 4.21.27\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24460\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-wjpjjj: failed waiting for cluster=\"np-version-upgrade-cluster-wjpjjj\" in resourcegroup=\"rg-np-version-upgrade-wjpjjj-pc5nbpkjrzdz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/28d75c45-25ff-4278-a246-ebcc78ca1ef0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/28d75c45-25ff-4278-a246-ebcc78ca1ef0\",\n      \"name\": \"28d75c45-25ff-4278-a246-ebcc78ca1ef0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:47:14.337423761Z\",\n      \"endTime\": \"2026-08-06T14:06:32.076160168Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-wjpjjj with version 4.21.27\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24460\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-wjpjjj: failed waiting for cluster=\"np-version-upgrade-cluster-wjpjjj\" in resourcegroup=\"rg-np-version-upgrade-wjpjjj-pc5nbpkjrzdz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/28d75c45-25ff-4278-a246-ebcc78ca1ef0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/28d75c45-25ff-4278-a246-ebcc78ca1ef0\",\n      \"name\": \"28d75c45-25ff-4278-a246-ebcc78ca1ef0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:47:14.337423761Z\",\n      \"endTime\": \"2026-08-06T14:06:32.076160168Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"063098a80cbfe3cc4638e71f1686605b92235136cfb5e8b62870202b46bdec68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"0b73f221e723983b7bb466871edd22f26516aca06559029bb0093f2257fe4ed6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"13acca32cc25889f23290d3862df2cd6cdd07bf1837b4014dbe05ff5e8eb9223","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"236fba08ee6513ef627e3344c7d63f6b4e48e780c89c1407f0125e055897a74d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"3a725768636b5db7a428db163677a754ba1bc010b17a1d104cb433e1bf1f102b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"43161b5c7fe5a04604aae7c86aad18d5fbd6ded8d83b993c6cb0eddb5c32ae32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"4c707ac016204c8e21bf0ad6090e04ec80173a388db068cdbe57c97cc89eb577","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"55cdb2e9385dc3b91f8cbacff5d9608844b3aa8edfe35103ec1619200bb37a21","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"5c03fe2235c8d06fb08c278dd75d56842305bea98635eb6b0cfc187099f924be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"5f977a303d985401d848e41a33b7dd9001fd63426df210f486aaa4f0f64723c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"631aff46b4ed57edbe2f280d536a16b29329f90aacc852a48bdb580e6b43319f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"6c295ab12b0ab36152d04a1401762b995e3b3de4d73fa6a738669845b30bbab9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"73b260c57d297290e7538f8366ba615328b5c703f508a1fe98a0b4ba6f52f797","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"9d516d7683d88b693761cce380780674bca380e614ab83ae644074e4f8409476","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"b6c6f188d72774218055eb6315f871d6aeebe7f1db640df0b66aec6dfcd558a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"d7d4632730204a64bbe64c7ed12cc3f61a340b62b2997f48c9c30126f44f6b2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"e2041a19de87ed544e987a8fc38139feb20ea47d9cf0d09871eb8eaead18deb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"ec7b912aaaba2ddbc3ce4c059ee6de5f7386b7169e4e245236d674d3fe8eafa3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-4-23-p9b9pl, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-p9b9pl in resource group rg-nightly-4-23-95wbldllmqlp, error: failed waiting for cluster=\"cluster-nightly-4-23-p9b9pl\" in resourcegroup=\"rg-nightly-4-23-95wbldllmqlp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-p9b9pl in resource group rg-nightly-4-23-95wbldllmqlp, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":9,"runs_affected":6,"failed_runs":6,"run_impact_percent":8.695652173913043,"anchor_occurrences":9,"after_last_push_count":9,"trend_counts":[0,1,0,2,1,2,2,1],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20","2026-07-27"],"prior_runs_affected":39,"prior_last_seen_at":"2026-08-02T09:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00221ac60\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-h6k2xdzb8p2t, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-h6k2xdzb8p2t\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-h6k2xdzb8p2t, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00221ac60\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-h6k2xdzb8p2t, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-h6k2xdzb8p2t\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-h6k2xdzb8p2t, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-nightly-4-23-xmwwqrjzhlmv/cluster-nightly-4-23-qtnlkx should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013051d0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-qtnlkx, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-qtnlkx in resource group rg-nightly-4-23-xmwwqrjzhlmv, error: failed waiting for cluster=\"cluster-nightly-4-23-qtnlkx\" in resourcegroup=\"rg-nightly-4-23-xmwwqrjzhlmv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-qtnlkx in resource group rg-nightly-4-23-xmwwqrjzhlmv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-nightly-4-23-xmwwqrjzhlmv/cluster-nightly-4-23-qtnlkx should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013051d0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-qtnlkx, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-qtnlkx in resource group rg-nightly-4-23-xmwwqrjzhlmv, error: failed waiting for cluster=\"cluster-nightly-4-23-qtnlkx\" in resourcegroup=\"rg-nightly-4-23-xmwwqrjzhlmv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-qtnlkx in resource group rg-nightly-4-23-xmwwqrjzhlmv, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002ad0390\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-snbsmkphx48p, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-snbsmkphx48p\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-snbsmkphx48p, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002ad0390\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-snbsmkphx48p, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-snbsmkphx48p\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-snbsmkphx48p, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2086588427117006848","occurred_at":"2026-08-09T23:00:22Z","signature_id":"17d0138a765333b9fc5cfb1ab63ea1221a2ebcc0b923e78b44c056f2115534e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2086226151444844544","occurred_at":"2026-08-08T23:00:49Z","signature_id":"3677014f901c50a032d707cbcf7dbe4c98b8545a357507ee74898a7698a5f52b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2086226151444844544","occurred_at":"2026-08-08T23:00:49Z","signature_id":"b7dc99c75988466df2b43c56a53aeae9a0db6e7d0e17ff899e24aa03f73911c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085863759292665856","occurred_at":"2026-08-07T23:00:48Z","signature_id":"52a1a0a1e4a456d0bb145d3a7be5b5c61a265faf42cc9e9f27e44eeec1c91c75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085863759292665856","occurred_at":"2026-08-07T23:00:48Z","signature_id":"f4a171315b92003b1607a0f8d0a186fd496d27b9dcb384c6640f0ae7edbbaf25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085501203734597632","occurred_at":"2026-08-06T23:00:07Z","signature_id":"9a5a1829f3ce94f2f253b46e318acac897ea906fab88e6b2b85a5ffdb6d75ecd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"7b6e9ec3c8f7a6ebd65015d4fc79091ff849f7f5770aff2cb7f75ded8aef079f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"ace94c99de58d994657790547c4d4456710fad10b9cc554190a334ba85d9a970","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084414011935297536","occurred_at":"2026-08-03T23:00:01Z","signature_id":"feff19a48ae22a841deeb9cfb524395a5a0f5b9a45e4dccfe66e45cb1f2b022a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":10.144927536231885,"anchor_occurrences":7,"after_last_push_count":27,"trend_counts":[2,1,1,2,3,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-13","2026-07-20","2026-07-27"],"prior_runs_affected":24,"prior_last_seen_at":"2026-08-02T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009ad7e0\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-z7fl6rs7cjg5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b557842c-42a5-4d65-80d1-401050420a8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b557842c-42a5-4d65-80d1-401050420a8d\",\n      \"name\": \"b557842c-42a5-4d65-80d1-401050420a8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T23:42:49.092257041Z\",\n      \"endTime\": \"2026-08-07T00:02:00.144502022Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009ad7e0\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-z7fl6rs7cjg5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b557842c-42a5-4d65-80d1-401050420a8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b557842c-42a5-4d65-80d1-401050420a8d\",\n      \"name\": \"b557842c-42a5-4d65-80d1-401050420a8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T23:42:49.092257041Z\",\n      \"endTime\": \"2026-08-07T00:02:00.144502022Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e90288\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-2tgsqq2jndsw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/60668361-99db-4918-b1e5-1dd336777928\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/60668361-99db-4918-b1e5-1dd336777928\",\n      \"name\": \"60668361-99db-4918-b1e5-1dd336777928\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T19:32:55.875583035Z\",\n      \"endTime\": \"2026-08-06T19:51:56.785806181Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e90288\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-2tgsqq2jndsw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/60668361-99db-4918-b1e5-1dd336777928\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/60668361-99db-4918-b1e5-1dd336777928\",\n      \"name\": \"60668361-99db-4918-b1e5-1dd336777928\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T19:32:55.875583035Z\",\n      \"endTime\": \"2026-08-06T19:51:56.785806181Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:174]: failed to create node pool \"np-zjfs89\" for cluster \"cluster-candidate-4-20-zjfs89\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d491a0\u003e: \n    failed to create NodePool np-zjfs89: failed waiting for nodepool=\"np-zjfs89\" for cluster \"cluster-candidate-4-20-zjfs89\" in resourcegroup=\"rg-candidate-4-20-lwxfdj6b82tz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b242787d-f97b-4793-82e7-f4a360d7884c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b242787d-f97b-4793-82e7-f4a360d7884c\",\n      \"name\": \"b242787d-f97b-4793-82e7-f4a360d7884c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:22:17.428651995Z\",\n      \"endTime\": \"2026-08-06T13:41:30.639404623Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:174]: failed to create node pool \"np-zjfs89\" for cluster \"cluster-candidate-4-20-zjfs89\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d491a0\u003e: \n    failed to create NodePool np-zjfs89: failed waiting for nodepool=\"np-zjfs89\" for cluster \"cluster-candidate-4-20-zjfs89\" in resourcegroup=\"rg-candidate-4-20-lwxfdj6b82tz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b242787d-f97b-4793-82e7-f4a360d7884c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b242787d-f97b-4793-82e7-f4a360d7884c\",\n      \"name\": \"b242787d-f97b-4793-82e7-f4a360d7884c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:22:17.428651995Z\",\n      \"endTime\": \"2026-08-06T13:41:30.639404623Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085501203734597632","occurred_at":"2026-08-06T23:00:07Z","signature_id":"4769652dcdd434db15b0f9f2f568efb4eb42cbef122eae222dda1c6c411aa2b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085439389160181760","occurred_at":"2026-08-06T18:54:30Z","signature_id":"16a035304592e71ce5eed76cf1a021ca48547d113592b6be98c79407f46e9e74","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"66db5f10f756daa0e52be78ac19d5c43ea8da173d9c8fd7bfd56943c2c79d417","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085099167385915392","occurred_at":"2026-08-05T20:22:35Z","signature_id":"e6cfd63ef72606f530614324528351264962e5d5d68bd93977952d6f44967019","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084975910284431360","occurred_at":"2026-08-05T12:07:16Z","signature_id":"9b85d0bd47f779fa46cfd7bc64622665a46f651236ae23d5f3ac14611ded8d9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084524966383980544","occurred_at":"2026-08-04T06:20:55Z","signature_id":"8aa340258f483bea6767d2949b250d007bd0b7d20c5700968c0934e4544adda7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084382205118779392","occurred_at":"2026-08-03T20:53:38Z","signature_id":"2dd0d606fedaa45e8dc4eb276ffe1c1538fb3f053a447b03bf757838ce13fc4d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":8.695652173913043,"anchor_occurrences":6,"after_last_push_count":21,"trend_counts":[1,0,5,1,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-20","2026-07-27"],"prior_runs_affected":21,"prior_last_seen_at":"2026-08-02T09:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084760764756791296","occurred_at":"2026-08-04T21:57:53Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084719557146054656","occurred_at":"2026-08-04T19:14:09Z","signature_id":"9f6f42bed68e6d3b86d81931d02f06532f74fc43be808bc0172db5fcca80bec4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084675180533125120","occurred_at":"2026-08-04T16:17:49Z","signature_id":"d712d5ce17264f0ef550496665e2a7bd7d67dfac479ff00deed5bcb5e610f7c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084479621713104896","occurred_at":"2026-08-04T03:20:44Z","signature_id":"c7ca409fabf5aaf248cdfd528d531467086d44fb1f0c733ecbbc0a31c3157986","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084477563748487168","occurred_at":"2026-08-04T03:12:33Z","signature_id":"d712d5ce17264f0ef550496665e2a7bd7d67dfac479ff00deed5bcb5e610f7c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":6,"runs_affected":4,"failed_runs":4,"run_impact_percent":5.797101449275362,"anchor_occurrences":6,"after_last_push_count":7,"trend_counts":[0,0,5,0,0,0,1,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-20"],"prior_runs_affected":5,"prior_last_seen_at":"2026-07-25T23:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba3a80\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-qb6d7sh7d2tg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c427c20-7cf4-403b-8cce-30c821f04345\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c427c20-7cf4-403b-8cce-30c821f04345\",\n      \"name\": \"3c427c20-7cf4-403b-8cce-30c821f04345\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-08T02:02:34.566633552Z\",\n      \"endTime\": \"2026-08-08T02:21:42.424942699Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba3a80\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-qb6d7sh7d2tg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c427c20-7cf4-403b-8cce-30c821f04345\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c427c20-7cf4-403b-8cce-30c821f04345\",\n      \"name\": \"3c427c20-7cf4-403b-8cce-30c821f04345\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-08T02:02:34.566633552Z\",\n      \"endTime\": \"2026-08-08T02:21:42.424942699Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed50a0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-g59rzx96ggqs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/30166c17-7166-438c-abb0-3f6693531389\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/30166c17-7166-438c-abb0-3f6693531389\",\n      \"name\": \"30166c17-7166-438c-abb0-3f6693531389\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T09:03:21.796374091Z\",\n      \"endTime\": \"2026-08-04T09:22:31.515288045Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed50a0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-g59rzx96ggqs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/30166c17-7166-438c-abb0-3f6693531389\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/30166c17-7166-438c-abb0-3f6693531389\",\n      \"name\": \"30166c17-7166-438c-abb0-3f6693531389\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T09:03:21.796374091Z\",\n      \"endTime\": \"2026-08-04T09:22:31.515288045Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-smj79nlv5zgb/cluster-candidate-4-22-7lcplf should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000992580\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-7lcplf: failed waiting for cluster=\"cluster-candidate-4-22-7lcplf\" in resourcegroup=\"rg-candidate-4-22-smj79nlv5zgb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/fb43d507-6da9-418d-9328-9479083254a1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/fb43d507-6da9-418d-9328-9479083254a1\",\n      \"name\": \"fb43d507-6da9-418d-9328-9479083254a1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T09:31:16.502576148Z\",\n      \"endTime\": \"2026-08-04T09:50:23.673134189Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-smj79nlv5zgb/cluster-candidate-4-22-7lcplf should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000992580\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-7lcplf: failed waiting for cluster=\"cluster-candidate-4-22-7lcplf\" in resourcegroup=\"rg-candidate-4-22-smj79nlv5zgb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/fb43d507-6da9-418d-9328-9479083254a1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/fb43d507-6da9-418d-9328-9479083254a1\",\n      \"name\": \"fb43d507-6da9-418d-9328-9479083254a1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T09:31:16.502576148Z\",\n      \"endTime\": \"2026-08-04T09:50:23.673134189Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2085909058149158912","occurred_at":"2026-08-08T02:00:48Z","signature_id":"1d86107f1b377d8c3e3b4a4902e4ae84baa11664368611b2717514bfbab74e7a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084564628381110272","occurred_at":"2026-08-04T08:58:31Z","signature_id":"299f99d007b42f44ccd643165d3abb7c6f6ecbc65bb99ed5ef68461b1d3619d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084564628381110272","occurred_at":"2026-08-04T08:58:31Z","signature_id":"330410bd24120063fe1b93743989d804569ee32f5b79931c1f9cccf564488270","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084524966383980544","occurred_at":"2026-08-04T06:20:55Z","signature_id":"bf0196d24a6bbaa1928d947a43911e1db713cbf805d29033684945b115ec996a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084477563748487168","occurred_at":"2026-08-04T03:12:33Z","signature_id":"284b6a9d47fef0affb83d2458655f4180c86861834bfab7ca08e3089c6a58c3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084477563748487168","occurred_at":"2026-08-04T03:12:33Z","signature_id":"aab9937daaf618bc4e1a5adc6d56082c5c1a6be307d5c3f47b2ed4468ecd1f0d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"24660fe0387a8714cdbcf9260ba9d42cd88ae6f8a2d64bc65cfcc351b75b07c4","failure_pattern":"ERROR CODE: NoRegisteredProviderFound; detail message No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: NoRegisteredProviderFound","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":5.797101449275362,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,1,3,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:120]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00043f880\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-n7rwkfwghfl5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/71021741-2fd6-4d28-a2bd-af27d69b5017\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:120]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00043f880\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-n7rwkfwghfl5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/71021741-2fd6-4d28-a2bd-af27d69b5017\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:120]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006c66c0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-dgjlznw6bk96\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/530dfc20-cb36-494b-a849-af72bb72b5f8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:120]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006c66c0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-dgjlznw6bk96\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/530dfc20-cb36-494b-a849-af72bb72b5f8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:120]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8e3c0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kmmrlf4fkxkp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/216af0ab-e97d-45ca-8bc0-fc5584177688\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:120]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8e3c0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kmmrlf4fkxkp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/216af0ab-e97d-45ca-8bc0-fc5584177688\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085863759292665856","occurred_at":"2026-08-07T23:00:48Z","signature_id":"a273f06173ccd3b55f575d770eab3ade69e9fe3e1a4a9f166f8ce6de1ed5978d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085598680915644416","occurred_at":"2026-08-07T05:27:28Z","signature_id":"bc2f29294896481c03bb8d8f808eb594600c2cee256b933ec026484d00d2df80","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2085546498744913920","occurred_at":"2026-08-07T02:00:07Z","signature_id":"5636737b43c4e80c3d131412afb0ba87e94189ad06ed731ad970258077163ef0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085501203734597632","occurred_at":"2026-08-06T23:00:07Z","signature_id":"7ba46c2f23dd67bf50b9c6f00c7043d6b6dbdd71455c7c119db41361c923025b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.898550724637681,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,1,0,1,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-tg2v4gkq6vnr/cluster-candidate-4-22-p7kqt2 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f10c40\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-p7kqt2: failed waiting for cluster=\"cluster-candidate-4-22-p7kqt2\" in resourcegroup=\"rg-candidate-4-22-tg2v4gkq6vnr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eb16ace1-2bdd-451a-9d78-7250d76ae24b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eb16ace1-2bdd-451a-9d78-7250d76ae24b\",\n      \"name\": \"eb16ace1-2bdd-451a-9d78-7250d76ae24b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:14:36.88824918Z\",\n      \"endTime\": \"2026-08-06T13:33:40.344451949Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-tg2v4gkq6vnr/cluster-candidate-4-22-p7kqt2 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f10c40\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-p7kqt2: failed waiting for cluster=\"cluster-candidate-4-22-p7kqt2\" in resourcegroup=\"rg-candidate-4-22-tg2v4gkq6vnr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eb16ace1-2bdd-451a-9d78-7250d76ae24b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eb16ace1-2bdd-451a-9d78-7250d76ae24b\",\n      \"name\": \"eb16ace1-2bdd-451a-9d78-7250d76ae24b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:14:36.88824918Z\",\n      \"endTime\": \"2026-08-06T13:33:40.344451949Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00113f260\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-w7wdnr2sm6ms\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c97f1841-2eb1-45a0-96c3-f9289da1a8a1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c97f1841-2eb1-45a0-96c3-f9289da1a8a1\",\n      \"name\": \"c97f1841-2eb1-45a0-96c3-f9289da1a8a1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T19:17:55.492730235Z\",\n      \"endTime\": \"2026-08-04T19:37:01.946103792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00113f260\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-w7wdnr2sm6ms\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c97f1841-2eb1-45a0-96c3-f9289da1a8a1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c97f1841-2eb1-45a0-96c3-f9289da1a8a1\",\n      \"name\": \"c97f1841-2eb1-45a0-96c3-f9289da1a8a1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T19:17:55.492730235Z\",\n      \"endTime\": \"2026-08-04T19:37:01.946103792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"b5a8313535e2abca1b3751a59805bec039aa97ee9b5c1005f43c575033f63ac4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084719557146054656","occurred_at":"2026-08-04T19:14:09Z","signature_id":"1cb51c0ff02d56202366bca5061ce9c661105ee6a8f1f17cc9ccacd63acf3b75","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.898550724637681,"anchor_occurrences":2,"after_last_push_count":11,"trend_counts":[0,0,1,1,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20","2026-07-27"],"prior_runs_affected":14,"prior_last_seen_at":"2026-07-31T15:09:20Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-6qxttk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e661e0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-46xsl.apps.aro.x2j7q6d3f2a2p2q.60fr.brazilsouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-6qxttk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e661e0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-46xsl.apps.aro.x2j7q6d3f2a2p2q.60fr.brazilsouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0xc002814a50\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-fpcgc.apps.aro.l6b6k4n3r7r3o9g.s2yb.australiaeast.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0xc002814a50\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-fpcgc.apps.aro.l6b6k4n3r7r3o9g.s2yb.australiaeast.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084975606633598976","occurred_at":"2026-08-05T12:11:36Z","signature_id":"9b26a8767aa4c9a1d3489fb2faa203a4bc9fd6e3348e63909580d96a47070c2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084479621713104896","occurred_at":"2026-08-04T03:20:44Z","signature_id":"a3864ffbd3e2d50315ed965f2fe4a8bbc310cd4905290c61b85b71f5f60d2b60","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a5df866d066b6d69aaea14be3ee0093ed9cfa1631a587d99b416a0bbe93a6074","failure_pattern":"v20260630preview API still not deployed as of \u003ctimestamp\u003e deadline","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:91]: v20260630preview API still not deployed as of 2026-08-07T00:00:00Z deadline","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.898550724637681,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:91]: v20260630preview API still not deployed as of 2026-08-07T00:00:00Z deadline\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:91]: v20260630preview API still not deployed as of 2026-08-07T00:00:00Z deadline"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085598680915644416","occurred_at":"2026-08-07T05:27:28Z","signature_id":"7269898a1029a630ccb580ccffd8c4ae1c6eb7659de2a27a8754827626d9330c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2085546498744913920","occurred_at":"2026-08-07T02:00:07Z","signature_id":"7269898a1029a630ccb580ccffd8c4ae1c6eb7659de2a27a8754827626d9330c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-candidate-4-21-4tlk2m\" in resourcegroup=\"rg-candidate-4-21-gwbm5ljx4jpp\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-21-4tlk2m in resource group rg-candidate-4-21-gwbm5ljx4jpp, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4492753623188406,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-06","2026-07-13"],"prior_runs_affected":5,"prior_last_seen_at":"2026-07-16T01:49:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:131]: failed to get admin REST config for cluster \"cluster-candidate-4-21-4tlk2m\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008ebb00\u003e: \n    failed waiting for hcpCluster=\"cluster-candidate-4-21-4tlk2m\" in resourcegroup=\"rg-candidate-4-21-gwbm5ljx4jpp\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-21-4tlk2m in resource group rg-candidate-4-21-gwbm5ljx4jpp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:131]: failed to get admin REST config for cluster \"cluster-candidate-4-21-4tlk2m\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008ebb00\u003e: \n    failed waiting for hcpCluster=\"cluster-candidate-4-21-4tlk2m\" in resourcegroup=\"rg-candidate-4-21-gwbm5ljx4jpp\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-21-4tlk2m in resource group rg-candidate-4-21-gwbm5ljx4jpp, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:120]: failed to get admin REST config for cluster \"cluster-zstream-4-19-crm6vw\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000939710\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-19-crm6vw\" in resourcegroup=\"rg-zstream-upgrade-4-19-2jfbcdvqrzbm\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-crm6vw in resource group rg-zstream-upgrade-4-19-2jfbcdvqrzbm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:120]: failed to get admin REST config for cluster \"cluster-zstream-4-19-crm6vw\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000939710\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-19-crm6vw\" in resourcegroup=\"rg-zstream-upgrade-4-19-2jfbcdvqrzbm\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-crm6vw in resource group rg-zstream-upgrade-4-19-2jfbcdvqrzbm, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"01c12e6c0d16de52443dd1ee73fd05c2e77edc7a6584fa351fe8a03ccbc410db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"dcc8d5fae49cd5abff9fecaec39fd8acf4d9b6b3cb487e5a440eb9cfb8074f9e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tkgzckfhvn9s, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tkgzckfhvn9s, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4492753623188406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20","2026-07-27"],"prior_runs_affected":15,"prior_last_seen_at":"2026-07-31T06:26:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000b2aa68\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tkgzckfhvn9s, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tkgzckfhvn9s, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000b2aa68\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tkgzckfhvn9s, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tkgzckfhvn9s, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"7245e5e986e4b456f99270b5e77b34f9c65e335493e3317ecedca65c0dcf9619","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f8c7c95dc98020682afa96f1eecf7f3fd55365a557c7372f0638f61b5f9d2054","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4492753623188406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-g5s7df6jfnm8/cluster-candidate-4-22-stc2rb should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e22ee0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-stc2rb: failed waiting for cluster=\"cluster-candidate-4-22-stc2rb\" in resourcegroup=\"rg-candidate-4-22-g5s7df6jfnm8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b19bfc54-b671-4c19-87b0-81e77f407bf9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b19bfc54-b671-4c19-87b0-81e77f407bf9\",\n      \"name\": \"b19bfc54-b671-4c19-87b0-81e77f407bf9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T19:52:11.833354934Z\",\n      \"endTime\": \"2026-08-04T20:11:23.470763058Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-g5s7df6jfnm8/cluster-candidate-4-22-stc2rb should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e22ee0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-stc2rb: failed waiting for cluster=\"cluster-candidate-4-22-stc2rb\" in resourcegroup=\"rg-candidate-4-22-g5s7df6jfnm8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b19bfc54-b671-4c19-87b0-81e77f407bf9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b19bfc54-b671-4c19-87b0-81e77f407bf9\",\n      \"name\": \"b19bfc54-b671-4c19-87b0-81e77f407bf9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T19:52:11.833354934Z\",\n      \"endTime\": \"2026-08-04T20:11:23.470763058Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084719557146054656","occurred_at":"2026-08-04T19:14:09Z","signature_id":"48b32c0cbe22ae7198ae23aeaf26f47fa518cff9f594ca286a939012bd570724","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3d6e4b895a78b4b03b1859653cdd20afe59584f0223f191cf548030518996633","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4492753623188406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-rqf9dr7wzqcn/cluster-candidate-4-22-ppxdfc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000880260\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-ppxdfc: failed waiting for cluster=\"cluster-candidate-4-22-ppxdfc\" in resourcegroup=\"rg-candidate-4-22-rqf9dr7wzqcn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/0432bd62-a18f-4bd9-b507-1577be3d81f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/0432bd62-a18f-4bd9-b507-1577be3d81f2\",\n      \"name\": \"0432bd62-a18f-4bd9-b507-1577be3d81f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T22:35:56.549310447Z\",\n      \"endTime\": \"2026-08-04T22:55:01.632384457Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-rqf9dr7wzqcn/cluster-candidate-4-22-ppxdfc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000880260\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-ppxdfc: failed waiting for cluster=\"cluster-candidate-4-22-ppxdfc\" in resourcegroup=\"rg-candidate-4-22-rqf9dr7wzqcn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/0432bd62-a18f-4bd9-b507-1577be3d81f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/0432bd62-a18f-4bd9-b507-1577be3d81f2\",\n      \"name\": \"0432bd62-a18f-4bd9-b507-1577be3d81f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T22:35:56.549310447Z\",\n      \"endTime\": \"2026-08-04T22:55:01.632384457Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084760764756791296","occurred_at":"2026-08-04T21:57:53Z","signature_id":"332803210855e6d9f65399232bf01dddf812673275a4becf3b8f160a11bf8d70","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"97958430e3a844b6f5be05213043088dddda4827bc264aa7b71a4bdf2cce51d5","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4492753623188406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-8ql88m2bbgwm/cluster-candidate-4-22-wnqnss should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d4a5e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-wnqnss: failed waiting for cluster=\"cluster-candidate-4-22-wnqnss\" in resourcegroup=\"rg-candidate-4-22-8ql88m2bbgwm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c09527cd-dc9e-4af7-999f-dc3c6c3a5408\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c09527cd-dc9e-4af7-999f-dc3c6c3a5408\",\n      \"name\": \"c09527cd-dc9e-4af7-999f-dc3c6c3a5408\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T04:07:58.639909171Z\",\n      \"endTime\": \"2026-08-04T04:27:02.537134643Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-8ql88m2bbgwm/cluster-candidate-4-22-wnqnss should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d4a5e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-wnqnss: failed waiting for cluster=\"cluster-candidate-4-22-wnqnss\" in resourcegroup=\"rg-candidate-4-22-8ql88m2bbgwm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c09527cd-dc9e-4af7-999f-dc3c6c3a5408\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c09527cd-dc9e-4af7-999f-dc3c6c3a5408\",\n      \"name\": \"c09527cd-dc9e-4af7-999f-dc3c6c3a5408\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-04T04:07:58.639909171Z\",\n      \"endTime\": \"2026-08-04T04:27:02.537134643Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084479621713104896","occurred_at":"2026-08-04T03:20:44Z","signature_id":"d446c23360bb3471487c32d5e9fd2debe1ecdd893ddfd53429d4d11b526a8087","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dcbb98b30046e6cc7914c7e62bede6790189f033e9f8bb7bf919eebfa6754c56","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: externalAuths, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4492753623188406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001d84300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-7zlftg2lbn8z\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/662e7bf9-43c4-4c69-b7d4-c84c5e38b0ea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/662e7bf9-43c4-4c69-b7d4-c84c5e38b0ea\",\n      \"name\": \"662e7bf9-43c4-4c69-b7d4-c84c5e38b0ea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:21:44.42927409Z\",\n      \"endTime\": \"2026-08-06T13:45:50.758225457Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s139v1qlkvi8boiqf5sq004bbeikr8v still exists (deletion dispatched at 2026-08-06T13:21:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001d84300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-7zlftg2lbn8z\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/662e7bf9-43c4-4c69-b7d4-c84c5e38b0ea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/662e7bf9-43c4-4c69-b7d4-c84c5e38b0ea\",\n      \"name\": \"662e7bf9-43c4-4c69-b7d4-c84c5e38b0ea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:21:44.42927409Z\",\n      \"endTime\": \"2026-08-06T13:45:50.758225457Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s139v1qlkvi8boiqf5sq004bbeikr8v still exists (deletion dispatched at 2026-08-06T13:21:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"765870913940abe2b981e937e3349b9a1e460508668804ddd9824f770fc9f076","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f6d43e4ab2fe017bbc54d307366c534735e562004e9108078bd8d55ba38e56fb","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4492753623188406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00171c228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-wvptdr\" in resourcegroup=\"rg-zstream-upgrade-4-21-bvlh6ndvpmbz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8813fe71-a996-402a-8b22-5a2a07f190dc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8813fe71-a996-402a-8b22-5a2a07f190dc\",\n      \"name\": \"8813fe71-a996-402a-8b22-5a2a07f190dc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T23:41:52.408330345Z\",\n      \"endTime\": \"2026-08-07T00:06:00.233442021Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s1cmcbdh4et6l3hlfi6mpmgpaik517c still exists (deletion dispatched at 2026-08-06T23:41:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00171c228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-wvptdr\" in resourcegroup=\"rg-zstream-upgrade-4-21-bvlh6ndvpmbz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8813fe71-a996-402a-8b22-5a2a07f190dc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8813fe71-a996-402a-8b22-5a2a07f190dc\",\n      \"name\": \"8813fe71-a996-402a-8b22-5a2a07f190dc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T23:41:52.408330345Z\",\n      \"endTime\": \"2026-08-07T00:06:00.233442021Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s1cmcbdh4et6l3hlfi6mpmgpaik517c still exists (deletion dispatched at 2026-08-06T23:41:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085501203734597632","occurred_at":"2026-08-06T23:00:07Z","signature_id":"2e5d7357142344a3caecc974d360925ce1edc5c37e5458306f994b1ef066b558","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a895eaf5fc00535af027e673197376a531131993383b5ea3abe908eddc6e9ca7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: externalAuths, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4492753623188406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001108210\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dkkn6llwn9ml\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a058d23-b76e-4230-8a1e-4b6d1a6bf1c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a058d23-b76e-4230-8a1e-4b6d1a6bf1c7\",\n      \"name\": \"6a058d23-b76e-4230-8a1e-4b6d1a6bf1c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:24:37.727370739Z\",\n      \"endTime\": \"2026-08-06T13:48:41.816178395Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s139utb7gn0spngljdld5adcoi9lknv still exists (deletion dispatched at 2026-08-06T13:24:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001108210\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dkkn6llwn9ml\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a058d23-b76e-4230-8a1e-4b6d1a6bf1c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a058d23-b76e-4230-8a1e-4b6d1a6bf1c7\",\n      \"name\": \"6a058d23-b76e-4230-8a1e-4b6d1a6bf1c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-06T13:24:37.727370739Z\",\n      \"endTime\": \"2026-08-06T13:48:41.816178395Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s139utb7gn0spngljdld5adcoi9lknv still exists (deletion dispatched at 2026-08-06T13:24:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","signature_id":"14899c226e729562a794009d5069ba2a91359831b77c6dd029233b500a2d34ad","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"384071bb4661dd33cc23587a2499fb7af7d344a9025c291e68dd800af8dd513c","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4492753623188406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cf6be8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vxhjwrv9wtgl\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c799ffec-de28-4c7c-9f3c-2074ebf9859b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c799ffec-de28-4c7c-9f3c-2074ebf9859b\",\n      \"name\": \"c799ffec-de28-4c7c-9f3c-2074ebf9859b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-08T23:30:28.852252888Z\",\n      \"endTime\": \"2026-08-08T23:54:36.968640243Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cf6be8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vxhjwrv9wtgl\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c799ffec-de28-4c7c-9f3c-2074ebf9859b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c799ffec-de28-4c7c-9f3c-2074ebf9859b\",\n      \"name\": \"c799ffec-de28-4c7c-9f3c-2074ebf9859b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-08T23:30:28.852252888Z\",\n      \"endTime\": \"2026-08-08T23:54:36.968640243Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2086226151444844544","occurred_at":"2026-08-08T23:00:49Z","signature_id":"079b61beb7f9f76241c5f4c73b44c97f2ca3ef00d4d00090dfce8d9de587b1ab","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"63a755b10a4ebc8c89991a40e36c8d3c0a9cd5e114a858de40038cb6cec08053","failure_pattern":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4492753623188406,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-02..2026-08-09 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-13","2026-07-27"],"prior_runs_affected":3,"prior_last_seen_at":"2026-07-29T18:28:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:230]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017d1410\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:230]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017d1410\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084524966383980544","occurred_at":"2026-08-04T06:20:55Z","signature_id":"487717933969c2d846f624aa14a8c8da25bd18563e3d8fae6d89725d3d3fb9d1","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}