{"meta":{"start_date":"2026-07-16","end_date":"2026-07-16","start_at":"2026-07-16T00:00:00Z","end_at":"2026-07-17T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T06:42:56Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":10,"failed_runs":7,"raw_occurrences":172,"matched_occurrences":172,"runs_affected":7},"rows":[{"environment":"prod","failure_pattern_id":"89cb7e06e525cd27136f4e938bd6f1470cdb9e65f768b9fa227d139fb3154e59","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-creation-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":82,"runs_affected":2,"failed_runs":2,"run_impact_percent":20,"anchor_occurrences":82,"after_last_push_count":121,"trend_counts":[0,0,0,0,0,39,82],"trend_range":"2026-07-10..2026-07-16 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b3ec00\u003e: \n    failed to create HCP cluster np-dg-minor-lb9dcf: failed starting cluster creation \"np-dg-minor-lb9dcf\" in resourcegroup=\"rg-np-dg-minor-lb9dcf-c67sqt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-lb9dcf-c67sqt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-lb9dcf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b3ec00\u003e: \n    failed to create HCP cluster np-dg-minor-lb9dcf: failed starting cluster creation \"np-dg-minor-lb9dcf\" in resourcegroup=\"rg-np-dg-minor-lb9dcf-c67sqt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-lb9dcf-c67sqt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-lb9dcf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00088b460\u003e: \n    failed to create HCP cluster ea-listrhx5jl: failed starting cluster creation \"ea-listrhx5jl\" in resourcegroup=\"ea-list-rf5hjq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-rf5hjq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listrhx5jl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00088b460\u003e: \n    failed to create HCP cluster ea-listrhx5jl: failed starting cluster creation \"ea-listrhx5jl\" in resourcegroup=\"ea-list-rf5hjq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-rf5hjq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listrhx5jl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ec5e40\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b2f8k8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-b2f8k8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ec5e40\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b2f8k8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-b2f8k8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"006ef6df674aaa68ee584052da9661b4ca37cbed28c131d3e0fa8b5630404e2c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"14313169092baa7f98810b8d7fcdbbba1a8056e70421db433b4fe2914a043fc5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"1c0159fbc68cc4b3cf75f225cbfcd26a1c351c9182da0fa149986531d77c8276","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"2c800ca1bbc26e52573b91fb627c021096d7ec54146105be79d301222ce09e42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"2e96a94221eba1e1173a6598f029ae04a1e9b92e1101d9f68f92e5479a4f1371","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"329c69fef69bb04c71f987c030537f0a70f394655e16b1992b4d0ca9e477c42a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"3ec2dcdf1a8dd783a9a3018dd0849c04be4d6ea4a11254cb9117d889129871eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"3ed85d1718a0101f764c6aaf7e3808c770cc057278a5d9aa1780e18a15696fca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"61f2155b3eeda90634a30247875559bcb5b5352bab61d695d71217242d8ea678","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"673b87f43b140686c3d45ea0a5172c743229e83dc499ab119efa35a0049b7326","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"6c2276e498ebb64ae9ee58f687579cd9d811b911bd0629e9787c8cb6d4b168e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"7ce5acdb2a56f387a1e0e8bd47db6fb579de23db9e07b0632e920bbe608ebd09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"7ee9d0458f8b0080146e06da8ee1a999f1b822478c62ccd4b3e286d536ae9e35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"8aa3973ac65dcacf1845c993375246b550213e63a213bd81154b67cf1119fa88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"8cef4b2af284ce38a593ec918bd10644d93a7984436f5dbb51260f17a2f06720","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"8f1b19f9a79e53095786f02e9be626cfba77b30a782168bef12afa43491f8400","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"90624b025c0c12e55a3fa25262dd921a0ff735ec7e6a54336cf4b2a1bc1d1de8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"933b693f94c52e27dd070222eaab8a45d664c5755cc26c2352f0b8e312b0cda1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"9b11160ace17341d26a1a3b3d01da249004f4293673cf369770bc874fb71fdd5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"9d6e8c263c4e1344f6d145f336395976fdfbe8c3da8d38fd54f85e5a7f3772b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"a59b4ffd2b0bf6eeaca2e29f5b777cd088d6527e954855c1628a608a7ef85b5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"a605bbee85ca886567330e8b825700bedc6ef788e34e94a3ee40f9728113370b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"a726694b62112acc5b2d655312c552a21f5272bcc34e47ed45fb525b12189e5a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"a9cb9c14edb5c6dbecb5a0688fe8ae8c80c28375b0b268b356a140ee2b295b8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"b86ef7b577b2c7d0704573cf846c7e4d3bbfd88d0aeb09c0e3715dde13dfc314","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"b9c7d981ce308509023aa59dc7917cc5a9b7b8c6d47fd9509a88ff2e02ed793c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"ba36b9308a029feb14e71e21e11e2b33a858d48a297b73abd05f386a2471c74a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"c10970fad00ad52d6b0fbac5e673529d396fee18ca1723a5aeb4236b842c2351","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"c17cf8c69d6f43b571e4200f4f7589eb3da8b746cac0e29666b4dcd074054abd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"c8664a517197fc75f77c90b91fbe6b64834f9f3efc649c76faf7bc7c237aed96","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"c9beafa36c7f015e30a206b402eb0032d8aa54b1ea0f438f992c80d924d5ca78","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"d4655e601d81ee5225b64440c73d8ccefb698fcb70c7ccc2ce0abcf50488b32b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"d95e229283c7468afe45130ac19293eb7d5123d89e3196d36a7608a8b4230263","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"df190ba88008b5dfbc50ba573fb6aa77141d2a1ed4e5d90b6a51fdcf6c9e2007","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"e60d67fa2e3cc8dfbf419a4467c12780174c8e45be8d59822fe2c58e4aaf2c25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"e726f20e42c6e654079dfcf01acfc6d703c8d567d4bf2daee63dcc1025376ad8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"e7d4184fd45236e072f062a6a089dcf02534d829afa2e77fbce4499cedd201a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"ed3ac10f965d943bffaafab5ae6c2973aa9090aaae5f0908aab94328dfd20148","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"fa749d05d9654281b1a0db192299082910d0a3c3d9c63269b8218d1cd856d5fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"00b5eae2c6eae9ad08b7f4045c6e998843c0fc3dcc1f39f3525ceb9f5205cd07","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"0d25f0b6cbf330a56ab0a71a93e63576f41dadcd417c14347ab761fa2c28746c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"16535d0f5304a32a284305295a4ed45f0e080e5413d5221682f7a2e5f0802c8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"1a20723e41424b46786c51fc87deabb603bcac0eba04519b3f5ee13f29dddf7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"21010e23c9ae002d9fd1eefb769f3b5c4a55170ea4d2a2f4d30399218d74058d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"24a64f288bacbe1662189b1a94012b71808217c6d4c2514e644bca8b15ae37ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"26da99f30b48a19bb4ed8afb5fafbbc175b6ab51d7adc9a3de0ac7d605920d79","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"34d897e0dd368dc985b7c260b00101ae2d95df4227dfe45a1b81c9f99e702bc3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"457c4e3a7c96ed47bd90e416652a7a34ec58512a2691a1014042e1c18bb1bcb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"4b2df19e2c5787db333f8ab39da3d0a7487077d1cca2e4384ff5330ac5199bf9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"4b9896d02e2e2f9e128c7ef8f1770eb5c76e008ed79665be9f67a5e6f993678a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"4de84d73884ac037c339354509a86797c9dc6f9b77c87df7d58d18c717b7dac3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"4f5f131f333eeb4a04acf7d589832c1dcaee5476db3969ad19945ab1bde107fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"4fb3153b7f5a7d2f5886d1701984c91f7539ae86dada7e529605dfdf99a06cfa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"504acf99694455792e2530ec1247020603de14a6bc942c44b439e7e218c91bf2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"5918184ca78ce551cc56388bc41adefd0da59f7e5f9c28b40597e777a8b8ac98","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"626dab9db7111cff398346aa55238f881276b6a21450babe9c9ab8912317d1e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"6272de87703632092028057bd09ff051ba82aca76fc85857d0118f361c23aaa2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"7d8f74a43ec97e55e50eeea4966a6185df72cf25f6da553f43f15b3d79ac9400","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"7df6f55970138e1a648dc3f3505b2d06327b2ad187108c68aa18765565fa2661","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"87e3e64dfdfb308eb1fc786aa71f96ce9581baff06eb3dea14aef10c5efb8681","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"88e020253456b46ed48a21888f04cd2d5971a0a420580d2270bfdffd9316ccac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"8bfb97806910827313106aa0f9d072a5f32a35aeb6381144b56c94a5b067a661","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"8f6ad892e0796c244cb1d210c3ca6298d3342972ddb6b0db47498398c46843a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"9c1c6327bcc2ec5c44962bdaa5976fe645b8650da9cee3a41c8fc9eb60c3e0eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"9e8ec3c008164b3e53956ddab529836c0155bf409dc4d6b0d029096f8e8ffe9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"9f3d07299782b7f9fcc84eb71a93ead55a85e7f1c65604705a31b4895d061e90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"a0467bd7e559500ce1c0edd06cdd1627907c05d4f9842e5f12be5f238b6e3b3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"a34fe6ef68e0ab85584490c94d1f884f7a13a00699aa2eada3a56bb61c4b82b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"aa2fe19bee2e55f0dc6b6352dc3210e1ca9dbeb0c1a50d075f3997bd3e2bbd42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"ad6a4d0b3c66c02e5eafb1c2f0c0483879689844d29889cb2a18f9bfbc028a9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"b066660cbc63f21f55c29a901f1106a56c8b83c6b9952b254cba9ea94b69d342","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"c20ff10818f806468f425c29911de3aa38d881db7e4448c3b189bf50706bc78a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"c39b3d35bd8c7e21d2e2d53eeb946fc64bca07e74e0691e1a02dbfee17512c19","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"c3fe9b59ce027be915c9004d7bb6e50138785b45a72a5d818bfa00cd498024e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"c86fb6c42a0519b0558d7e1cbda2920684b79938147ced829f83a82626df5d1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"c8b95f1681b829652013c1c1a4f23206d433e55bfcdca66294f98c363495e179","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"cb1206f709fb338f70195b31b91d2daa806970ceef6e7ed58ae0acbcec6772e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"d4deaa5fd8efb8ba289dc83e1db609ec48903dbc4339786b4b23526ab90fe1de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"e8c789188534b0a9eab595c76ca5c811d5002cd217c0cb88858093853e1a3c6e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"e9555dbad207ee75100e84268c5963a4e1ba59235848a45ae09bc5465cb208e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"e9e9d923f54ac93fbfa314687e9396ca717f0b554bbb195d2c8a1f2360239c15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"f11e7d267b169b2d8fe976bef63531e0d7204a1cb572b12795b104d9cf8016f9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-27zt4d, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-27zt4d in resource group rg-candidate-4-20-5bwqvr, error: failed waiting for cluster=\"cluster-candidate-4-20-27zt4d\" in resourcegroup=\"rg-candidate-4-20-5bwqvr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-27zt4d in resource group rg-candidate-4-20-5bwqvr, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":61,"runs_affected":3,"failed_runs":3,"run_impact_percent":30,"anchor_occurrences":61,"after_last_push_count":497,"trend_counts":[30,46,0,0,0,186,61],"trend_range":"2026-07-10..2026-07-16 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-15","2026-06-22","2026-06-29","2026-07-06"],"prior_runs_affected":57,"prior_last_seen_at":"2026-07-11T23:00:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:111]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012bee40\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lbqd48, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-lbqd48\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lbqd48, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:111]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012bee40\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lbqd48, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-lbqd48\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lbqd48, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-22-dchmmr\" with version 4.22.3 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a1e9c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-dchmmr, caused by: timeout '35.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-dchmmr in resource group rg-zstream-upgrade-4-22-922f8s, error: failed waiting for cluster=\"cluster-zstream-4-22-dchmmr\" in resourcegroup=\"rg-zstream-upgrade-4-22-922f8s\" to finish creating, caused by: timeout '35.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-dchmmr in resource group rg-zstream-upgrade-4-22-922f8s, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-22-dchmmr\" with version 4.22.3 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a1e9c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-dchmmr, caused by: timeout '35.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-dchmmr in resource group rg-zstream-upgrade-4-22-922f8s, error: failed waiting for cluster=\"cluster-zstream-4-22-dchmmr\" in resourcegroup=\"rg-zstream-upgrade-4-22-922f8s\" to finish creating, caused by: timeout '35.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-dchmmr in resource group rg-zstream-upgrade-4-22-922f8s, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014926c0\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-v6sxdj, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-v6sxdj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-v6sxdj, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014926c0\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-v6sxdj, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-v6sxdj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-v6sxdj, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"013e289d65caf1ed404e4e6bb11d936eb52cce6d41d7effd58f98bb6970dbc94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"052fe3b53ed222992b95fd900a6a089c812ae8bbf3943ba5302fd18f791922bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"08690bdd58d54d52f6b361de8974a4aefaaf6fe9ff18eed63d057476a2ff0bea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"0dec227fa775a1a86f8a261cd6797de4bdca40a071cd9668b653bb1e2c798459","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"0e7a1ddd0dd6b8f1b65d831cebf3829d289ab489692059494071a54bfd98c74c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"102ac0d7d49188429dbdd3f212bcb477494c357e5a220097c0acc6dead09b3b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"1a88e99e6a0455b644033c8ca3e02c9a935a1e5e41d85dd818329f2985e390c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"27967ab5b933fce6836be9314de023d1776f77f36db5a53b40cb5e56a4a1a5a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"2d9a5142623f1dd6a719ae3da502c185992e225e3ef3951822aaa92575d33220","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"35775642a6050a373f3daabcd7d4ee047c0f2a5d1b973540988d39de35642afd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"37f323d22b0fba140a69cfe3ae69f4191a6d8de702342e2b3eb36c573212afe9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"38191a6d510a8e8893c0aa457ec196e5d97ef215124908fafd2519c0423c16be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"3ace77ca21aefc5c9655049c1d838d0ae1c234cb3ff7d00d673d770e99022251","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"3e835a4d50271720f0b65c51a99d81579d32a707f87adb93261a8901d79875c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"406745b256b734c1bbc726b55b42fbdf305354a44b05102c72a40614fe0f8f01","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"46d93b101e68816e53cfe94c7c199657a1aa6b981bb39a474b4b268b1910aab5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"4fa22593b476021fdaf419d16cc27dda5177d128d77c749b6a7722edef568089","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"561a0b7f34261a7add2eb34d5673e8c426cb5be1cb751bb77e09526ecb2e5da0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"59fb00cde73a61c49fbb649a0a9832c5a2fa27cf0f21a9be5bd8da4f62007302","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"5cc344f0c4908a6f5580c39c38c56d2fd8567bd74a2c574630ba6a5dd7306df6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"5f7b55563d53c7a2606c13011f3e3e065887081b916e8972753566924e23805b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"67a08b4e686ddd109c9a76a10acce91e42b7102077ed6059eab588b947e2a38e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"768ac9f1584724904a1f64fcc09000071277c2606ff3610a7ade71ce20afe74a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"7e4861c38031e762cd3b5fec2bc13a194f6abfcfdbfbee48a4270cba766f955a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"96b69b76e6d375ac014b8ec3f16e53c10962bdcc218f2fb832909a6ea0153e47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"9897dceb77a7b258a9a08c58ff5fffec73fa66c34f4f0cfd7fe385c5ea890bd7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"98c83016e505277192e79e947cbfe9e4449891bc9434b7e11f375a9eab95f3c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"9eb2cb05939be4e87a6ce944ef5cb44ac2ba85132eed03f845ae9c01c48ef2d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"b54177d5da6b1b3813d43cef400462de6c5544732df9e3f0d496eee91ebb973c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"bcbc95cd2c8b22cdc8288ec64822ffe4e8cef54997ab88136d042235c7058e06","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"bf4af766df169003f3813aad8c8bc07664fa7fd50d9ad12dcb7fe1675dc04d7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"bf890a33bdcdaa5154eb3ba5c16b7f98697cb2cc9b1e1288e6d12dfc2bfa09ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"c31e38ea4019a37c7a0ebb462da0f72673c3dfc47880da2d9f182ef5df7c95ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"cae624e91ae34e636385aad5c56324b7dcbfd2133a8056182af3441b438b7661","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"cc6d03f5423363bd2238dc7a44bb18b64a8d6981f86d4dd67bc4703acf855b69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"ce9cdf05e0f718117412a7bd4841a713973265a3efaaf3dcdb26606d9737aa4a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"d021eba54c3b9e5c60630569f24364b5f4373698e1e419ee3535a1fed254486d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"d3449754506df5ad1ffba667adcf3adad11a06ef0fb8d3e8dcf3d8c047abdace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"dfaba4c0095384179e3764126f031aa14b4d745b6bfebe188d57a38d61555a4d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"f2869c096a7881db64598120565ce57e4be61447867a5b3a747bcf26a0cb5f5b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"f8aa8195c030798314c17bae45ba67030119e7df70b626b80cdf970af4988afe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077575353861148672","occurred_at":"2026-07-16T02:05:38Z","signature_id":"a340def35a6f37808f4050637f1016d8e3c52d436854cbabfbb1da40697ded47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"0d17d03c052f693515a115ffb028a096acfb0edfddab6704087651ba3b3788e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"40572a1a58e8afba40f306abd71b3d3aca411367ad255c55e6aa62edc6b10d92","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"49ae78b66c26d0ab8ef5cd639bf0ad1859ed3e7548c2c249ce5816f42f7dc567","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"4d952dc61830d1356f0f9b50d39a30fcb5769b32f84215c139885ac1e32a86c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"5eff10b7672e87b26ae5ed6991d752dcb0eb27c6eb7224ea5334af357e6be892","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"6416700c5cfc1e7d65890dc09ab9c06b237934489f8e288358b297b930312e72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"745c19eaf826fbaa02e6b96375b5b69e7dc08026a11af352c6bfd55f5039faeb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"758f6dac0e7f1c0e03c156576241afa1ce13fc7f5bf9183a17da2f9526330b92","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"8a4a83492d1d8c27c0025792a3764eba5d511535bd42796008acaed213e640a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"9cfd2a0e93f2acb47f510eadaa111b1fecb594d7c0eee62afdef92d21a5dbf59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"bbb4027c13e87587566cbb9d9c71d92ded766f14e94705e84c4582d4393929ba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"be0b18bbba99548e2491b55986f2d330ba43e3dc8196bfae7c4b6898caade88f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"be3b126e720355487373fa0c9d2657a57dc2359c8e02e91abf0dfe6b3c7b0978","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"c484448308a46cf30b32ed22cfa03fa72c3e4f71e3cb4229a9c5359734c9f734","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"ccf4a16effa8552fc0def2eb11e24e7d7267bd15a2e48e29af9617bd24270b75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"e7030ff2e4c465340fad67f3e2d5dfed1934c7576ca67c84e25b31f6138e42ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"e76921eea079c8cd9ab1ba9b6ecf2ca71aff2d11bbd8fe9cde75b3e42baf1487","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"e797d6c06edaa51d0064f221eb2ec83d51ac9977e8ffdce0b54c5227579da25b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"fb2f23fbe2511baafc9169ff80b4ddee98fbf84ff88c70637fa2c427b8867e1d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-fpjt95\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-5gwtfb\" in resourcegroup=\"admin-credential-lifecycle-test-fpjt95\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-5gwtfb in resource group admin-credential-lifecycle-test-fpjt95, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-5gwtfb\" in resourcegroup=\"admin-credential-lifecycle-test-fpjt95\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-5gwtfb in resource group admin-credential-lifecycle-test-fpjt95, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":15,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":15,"after_last_push_count":25,"trend_counts":[6,0,0,0,1,3,15],"trend_range":"2026-07-10..2026-07-16 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-06"],"prior_runs_affected":6,"prior_last_seen_at":"2026-07-10T23:00:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000b9e3d8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"idms-79lt56\", caused by: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-79lt56\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-79lt56, error: context deadline exceeded, error: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-79lt56\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-79lt56, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000b9e3d8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"idms-79lt56\", caused by: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-79lt56\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-79lt56, error: context deadline exceeded, error: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-79lt56\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-79lt56, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc0025dc120\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"clusternp128-rw8vb2\", caused by: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rw8vb2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-rw8vb2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rw8vb2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-rw8vb2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc0025dc120\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"clusternp128-rw8vb2\", caused by: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rw8vb2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-rw8vb2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rw8vb2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-rw8vb2, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc001690330\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-zstream-upgrade-4-19-2l4sb9\", caused by: failed waiting for hcpCluster=\"cluster-zstream-4-19-hs5k7l\" in resourcegroup=\"rg-zstream-upgrade-4-19-2l4sb9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-zstream-4-19-hs5k7l in resource group rg-zstream-upgrade-4-19-2l4sb9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-zstream-4-19-hs5k7l\" in resourcegroup=\"rg-zstream-upgrade-4-19-2l4sb9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-zstream-4-19-hs5k7l in resource group rg-zstream-upgrade-4-19-2l4sb9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc001690330\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-zstream-upgrade-4-19-2l4sb9\", caused by: failed waiting for hcpCluster=\"cluster-zstream-4-19-hs5k7l\" in resourcegroup=\"rg-zstream-upgrade-4-19-2l4sb9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-zstream-4-19-hs5k7l in resource group rg-zstream-upgrade-4-19-2l4sb9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-zstream-4-19-hs5k7l\" in resourcegroup=\"rg-zstream-upgrade-4-19-2l4sb9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-zstream-4-19-hs5k7l in resource group rg-zstream-upgrade-4-19-2l4sb9, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"12e11d957fad9e3d6117efa74d8b90bd49f8cbb1fa0bdd31e94dbdaed6625fb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"170e4ea72ad58f13dded240dc914872eec44aefb63e0f8561e833c243ea4bcc2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"3d6887031c7db06d2afe1c1e38bf87f2b14237edd86a022c196ba0379fd21a60","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"40221ba4c72c77f1582117c2ed43e9a751020b9edfac0976ab8ef786978f5323","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"41771f77e99bdca69ab727ce2e145b483c58e209b5fdf7b776e756a35eb99596","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"551f70fedf5d3aa05dbcd6d375a62f2d10937c4ccd232193dabddb83e5da506c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"55a833b489b2e93c71a76ff60d7561d81966d1054974a8d001114695baac2dd2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"9207453d432a2f2aa8ff6dd0dcfd8f7271ce85f580b0567ae6c2ec54fd5d2bfb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"96c55f5247bcb9c487bb22340df42abeea62186f23461cb75163e2497360c4d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"9cc9f8f1700a7fd52488651ad5a3bde38dff2d03ede86b336162a6feff8d5323","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"a64f7322491bec37008c4078189dcbeb55d06bbad9ea515ebc8d45629b7868f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"a78db7c25cf5e854e389182efdadd356a70e72035407b1af1530b9bc37d30dee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"cf664694b37a8dd13d3717b81858b895f778f83f11045649f547ffabbc6aaa32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"d16b3aca732d737225a8167eff17303370688fefef788c6b83bdc550e793da47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"d1d4a2ed50c6fdc60e50e308b953242a503fdfb764ac1ecc5866eb17449c896a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-2qtzzw, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2qtzzw\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":30,"anchor_occurrences":4,"after_last_push_count":13,"trend_counts":[3,0,0,0,0,0,4],"trend_range":"2026-07-10..2026-07-16 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-15","2026-06-22","2026-07-06"],"prior_runs_affected":24,"prior_last_seen_at":"2026-07-10T04:27:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-rslfbh\" for cluster \"cluster-candidate-4-20-rslfbh\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002438480\u003e: \n    failed to create NodePool np-rslfbh, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-rslfbh in resource group rg-candidate-4-20-qk525f, error: failed waiting for nodepool=\"np-rslfbh\" for cluster \"cluster-candidate-4-20-rslfbh\" in resourcegroup=\"rg-candidate-4-20-qk525f\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-rslfbh\" for cluster \"cluster-candidate-4-20-rslfbh\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002438480\u003e: \n    failed to create NodePool np-rslfbh, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-rslfbh in resource group rg-candidate-4-20-qk525f, error: failed waiting for nodepool=\"np-rslfbh\" for cluster \"cluster-candidate-4-20-rslfbh\" in resourcegroup=\"rg-candidate-4-20-qk525f\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-qcmzfw\" for cluster \"cluster-candidate-4-20-qcmzfw\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002808cf0\u003e: \n    failed to create NodePool np-qcmzfw, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-qcmzfw in resource group rg-candidate-4-20-hrn55j, error: failed waiting for nodepool=\"np-qcmzfw\" for cluster \"cluster-candidate-4-20-qcmzfw\" in resourcegroup=\"rg-candidate-4-20-hrn55j\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-qcmzfw\" for cluster \"cluster-candidate-4-20-qcmzfw\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002808cf0\u003e: \n    failed to create NodePool np-qcmzfw, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-qcmzfw in resource group rg-candidate-4-20-hrn55j, error: failed waiting for nodepool=\"np-qcmzfw\" for cluster \"cluster-candidate-4-20-qcmzfw\" in resourcegroup=\"rg-candidate-4-20-hrn55j\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:153]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002261ec0\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-2qtzzw, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2qtzzw\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:153]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002261ec0\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-2qtzzw, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2qtzzw\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077626877937192960","occurred_at":"2026-07-16T05:30:22Z","signature_id":"397ba042c302a7fe07f9f0da6894c6c1fea13fe400f0544c24c639591453f61b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077575353861148672","occurred_at":"2026-07-16T02:05:38Z","signature_id":"06c286ce86e95ddc1c4e3958843699750f38e7b614d83877f2ce86c6869865ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"084f7a189a36dcb30ff0b8d8ff98c6a065ef4d1cc7ff43e6f533c05d318bbcf1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"5652adad6f72cf49f1e056a8fcef9666efd3ce7e0f09d5702a911cf7af6f86be","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7ac9ea1609c23f13fc59d25bc0226d6ddbb91e9217b1f67c5624f53eb77e8333","failure_pattern":"error should indicate FIPS value must be true or false","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":20,"anchor_occurrences":2,"after_last_push_count":12,"trend_counts":[0,0,0,0,0,1,2],"trend_range":"2026-07-10..2026-07-16 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-29","2026-07-06"],"prior_runs_affected":12,"prior_last_seen_at":"2026-07-08T02:00:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2077891050780758016","occurred_at":"2026-07-16T23:00:06Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2077574106777128960","occurred_at":"2026-07-16T02:00:41Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79232f7fe5f1a65f3af30b7ba05ac2a1643a1a699ce682f3bc11178c03f0cbf4","failure_pattern":"failed to wait for first cluster \"\u003ccluster\u003e\" to complete creation (timeout '20.000000' minutes)","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":20,"anchor_occurrences":2,"after_last_push_count":11,"trend_counts":[0,1,0,0,0,4,2],"trend_range":"2026-07-10..2026-07-16 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-15","2026-06-22","2026-07-06"],"prior_runs_affected":13,"prior_last_seen_at":"2026-07-11T02:00:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"fa82b432fd7a36953b4c71f37791bced4d6db1832370f07a5c77508bfa36ef7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077575353861148672","occurred_at":"2026-07-16T02:05:38Z","signature_id":"fa82b432fd7a36953b4c71f37791bced4d6db1832370f07a5c77508bfa36ef7e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-l68qpz\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l68qpz, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[2,0,0,0,0,0,2],"trend_range":"2026-07-10..2026-07-16 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-15","2026-07-06"],"prior_runs_affected":5,"prior_last_seen_at":"2026-07-10T23:00:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:150]: failed to get admin REST config for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0023fe8d0\u003e: \n    failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-tcgqq4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster private-kv-cluster in resource group private-keyvault-tcgqq4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:150]: failed to get admin REST config for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0023fe8d0\u003e: \n    failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-tcgqq4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster private-kv-cluster in resource group private-keyvault-tcgqq4, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: failed to get admin REST config for cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00109da40\u003e: \n    failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-l68qpz\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l68qpz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: failed to get admin REST config for cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00109da40\u003e: \n    failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-l68qpz\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l68qpz, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"a7e3f8b63fe3c385b17b7ba1a293aa6a1f668aa676ceb02e400c24b1ff652399","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"bb14065affbc74734d828b16bc3bf596687e6ae7017f9f6452f5ed6e68857eda","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dcad60eb8db626db10bfb0812a18e7c392049987a81c4fcbcdc5140e0032dc41","failure_pattern":"Cluster should become ready within '20.000000' minutes","search_query":"Cluster should become ready within '20.000000' minutes","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":12,"trend_counts":[1,1,0,0,0,5,1],"trend_range":"2026-07-10..2026-07-16 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-15","2026-06-22","2026-07-06"],"prior_runs_affected":14,"prior_last_seen_at":"2026-07-11T02:00:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.000s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.000s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"89a8869f14426e34389e05b72b79a4a780586c738b83849eed78d8560ca7ff87","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"13d75a9f9f99d70a6d085901dd667b5d97db992be391e6dfa70f303d1348de7b","failure_pattern":"ERROR CODE: DenyAssignmentAuthorizationFailed; provider Microsoft.Authorization","search_query":"ERROR CODE: DenyAssignmentAuthorizationFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-10..2026-07-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000dd6bd0\u003e: \n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-d424ms, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-d424ms\" to finish creating: context deadline exceeded\n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-8597d-29b5z\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-8597d-29b5z\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8597d-29b5z/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: DenyAssignmentAuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DenyAssignmentAuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' has permission to perform action 'Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action' on scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8597d-29b5z'; however, the access is denied because of the deny assignment with name '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed/providers/Microsoft.Authorization/denyAssignments/30157864-4425-5338-9fbd-2c478c8dd9d7' and Id '30157864-4425-5338-9fbd-2c478c8dd9d7' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n      - VM \"cilium-cluster-cilium-np-8597d-7fmlt\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-8597d-7fmlt\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8597d-7fmlt/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: DenyAssignmentAuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DenyAssignmentAuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' has permission to perform action 'Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action' on scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8597d-7fmlt'; however, the access is denied because of the deny assignment with name '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed/providers/Microsoft.Authorization/denyAssignments/30157864-4425-5338-9fbd-2c478c8dd9d7' and Id '30157864-4425-5338-9fbd-2c478c8dd9d7' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000dd6bd0\u003e: \n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-d424ms, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-d424ms\" to finish creating: context deadline exceeded\n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-8597d-29b5z\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-8597d-29b5z\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8597d-29b5z/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: DenyAssignmentAuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DenyAssignmentAuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' has permission to perform action 'Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action' on scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8597d-29b5z'; however, the access is denied because of the deny assignment with name '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed/providers/Microsoft.Authorization/denyAssignments/30157864-4425-5338-9fbd-2c478c8dd9d7' and Id '30157864-4425-5338-9fbd-2c478c8dd9d7' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n      - VM \"cilium-cluster-cilium-np-8597d-7fmlt\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-8597d-7fmlt\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8597d-7fmlt/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: DenyAssignmentAuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DenyAssignmentAuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' has permission to perform action 'Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action' on scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8597d-7fmlt'; however, the access is denied because of the deny assignment with name '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed/providers/Microsoft.Authorization/denyAssignments/30157864-4425-5338-9fbd-2c478c8dd9d7' and Id '30157864-4425-5338-9fbd-2c478c8dd9d7' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d424ms--managed'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077571410238115840","occurred_at":"2026-07-16T01:49:58Z","signature_id":"d6a9cb6c90a51fef739e198edf5a677ab88313e9b38a6da4edc9028283d5e604","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"471f22e5967d4c6d260358df80d5f1c22cdca60ff8d89437ff00486c66a994f2","failure_pattern":"cluster has not yet reached Succeeded state","search_query":"cluster has not yet reached Succeeded state","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,1,0,0,0,0,1],"trend_range":"2026-07-10..2026-07-16 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-15","2026-06-22","2026-07-06"],"prior_runs_affected":10,"prior_last_seen_at":"2026-07-11T02:00:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:217 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:217 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077700502522433536","occurred_at":"2026-07-16T10:22:56Z","signature_id":"347c1e6b5b75b4869e1612ebdf39eb29ba4968297ea24157167a6ca6c0c772c8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-10..2026-07-16 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-15","2026-06-22","2026-06-29","2026-07-06"],"prior_runs_affected":8,"prior_last_seen_at":"2026-07-09T23:00:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-mhk487\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cd4360\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-2rvs6.apps.aro.j1c1u0t2q7v0b4j.gq5t.canadacentral.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-mhk487\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cd4360\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-2rvs6.apps.aro.j1c1u0t2q7v0b4j.gq5t.canadacentral.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2077680124106379264","occurred_at":"2026-07-16T09:01:00Z","signature_id":"df7523bcd0678036414616f616795886354b6a8313b98cff543056fc139fc076","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}