{"meta":{"start_date":"2026-07-18","end_date":"2026-07-18","start_at":"2026-07-18T00:00:00Z","end_at":"2026-07-19T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T05:23:56Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":4,"failed_runs":4,"raw_occurrences":152,"matched_occurrences":152,"runs_affected":4},"rows":[{"environment":"prod","failure_pattern_id":"89cb7e06e525cd27136f4e938bd6f1470cdb9e65f768b9fa227d139fb3154e59","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-creation-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":82,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":82,"after_last_push_count":328,"trend_counts":[0,0,0,39,82,125,82],"trend_range":"2026-07-12..2026-07-18 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00074be40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-wns6k4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-wns6k4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00074be40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-wns6k4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-wns6k4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00085a020\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-9gj9hw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-9gj9hw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00085a020\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-9gj9hw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-9gj9hw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-d4qbkv\" with version 4.20.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000471ee0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-d4qbkv: failed starting cluster creation \"cluster-zstream-4-20-d4qbkv\" in resourcegroup=\"rg-zstream-upgrade-4-20-g6q587\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-g6q587/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-d4qbkv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-d4qbkv\" with version 4.20.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000471ee0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-d4qbkv: failed starting cluster creation \"cluster-zstream-4-20-d4qbkv\" in resourcegroup=\"rg-zstream-upgrade-4-20-g6q587\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-g6q587/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-d4qbkv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"05e6e1f082d91e093a3608df64b424c819f5ce82e388bc31545ab18fc6ccccd8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"3838d82f07754d0ce25bd35178584b47d956cdb80108495de3d0b70e07dd1ef9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"41893077ea414df57a5b7cdea6688db9a2fd155c4286d62a4ae7a9f615bc9a3b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"479652d60957278028cdc492c7799c3e2fda29ab0e2fa49521d4c6e53956cfe6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"5c3d8ab2411af21c04a63441ab100cf87cda43cd7913e0873209867da2e6a4cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"6bc3368e5a9d447661f63d2fe399235cc49913cd201a7b3813d707d1bcc3e8ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"7a58d1a9b79a7309a85983c4d844d6105b090027a88b8612a3faadee039de595","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"7b5d1e7f891c469a1718f83682f7ccef8f12891c1bcab5923c4870b4223b2fc2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"7bb1413a36638bd3ca7c89cefc47fa31c192ca7eef82cad0c483051012226ccb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"7c4324843aece3983d6945a437fadbf5d0f1dcd700853ec10963de7c023aaee0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"7d4dee1dabeff1f124f5f950f44501c70f2569d2efb7cbc0161a62bc2e3ffeb9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"82cdba6ab042670575e874f37f3a35b94d72d39f8ce45c80901ae2c1f9e14e2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"859e65fa4de4d5602deb31a7a01adcf01deae8f6dcf72fa77d0df2bc6bdb4faa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"8c7711f575007ea2331c0e97106f4eccd4a895849a74cb98a518f592288ed90a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"8e71b5a4f89b349a7800f437ac71320b1ed98f8223672edc2dfd13c98bea432b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"97b2381ba2a458cf0dd81fb14937dd911b0c8341ad74a8dbce467ec20e72777e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"98955a1520fa35470699b7c4111cb34bb6de035fe772a049a0343cab1c944dae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"9aa60afb86f0be3b0ead7c82590e57b292abf25ca484474a1492b6e04a22dc12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"9ac7e9f46d4015e5bd1c6a07c4fe3127b701b9e51728ee177d4cc0c191f8bfe6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"9d9182afa2373be8977b04a442cd9e5bad14be6886a1da3efb1be22c992f874e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"a0765157e6316828f0bc7c54fae427c4b9c8b791c54f1bf8005d8c483aaa8d3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"aea04fc8430fc9581407444d21f2a7f1b3838a81cf291f28861dd1c635215116","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"afaf6264522bc86c1596a77961adf21c38249f92705d5e84280e40ed6de61899","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"b27159614ff835090b491908b2084c716481d578ac730b264e83d13b2b8f1766","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"b954bbeb08661f1b557eced3c28b00a5ca1aca4dc2347bc585532a2140730587","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"bcee8cf22d4de4e9724627a22ddc616584ece7b4564171bf5caf90718727c3ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"bf61528179d0b16cad19e210e682b52e8b5063708ede6b8fdfe873fb4f3469b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"c34823c2426dfc47d1988dc6fc31d7b54e925d592bd66f0adbbbfa3ab1575767","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"c658f7822ffa4a6d081c2a2d3c6977a2f3d1c49df42e1dfdb6a4866eb460d09a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"c85f8ba4c65e587c1c0323f8fb38dfaa3c0307fa46874796c9bf2df8f7b79910","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"d89b199832cd1f9baba1505884d5dba56d8c1c3cb065011198ee1c3d8605a505","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"daa8eb653cd8cbc87ad813ed83bb18578d4d9884eb5d94964f26386cdc458f1c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"e24995197f3831020fad6b116a147f2d8d7099161d718d2adfcf331c7b697466","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"e5b1ab7d9de1939ec39b8e9ff20455ef622eed123e93e961e501d7495820aeee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"e841c38c930170d1bf65d3048fca5a897f81e95b807b48fc7f71982fc7ea64ba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"f0050cea5fa423923e30efd888df93fe41542dfc1186ffc3bb6d61aef3315ee0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"fb3b072eae79c1bb7195d6e60eefc126da9463f8ae4e9ebab701af4647d67b23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"fc16b97774517fcc788cb06912a9abde3832cf5917013abd0b9d76bc699e03d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"feaec33f1789208daed7c056f75349a4ab3bccedfe6d9f05fad4ce225e10f1e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"016b5b88ffbf0f9705427d9ce1c815efcf9d42b3a5fa5095a3993d515e6a83c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"067951e89f54d74178ff0cd32adc21eec5b5074a4f22cddcf1f0c95d67d5723b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"090c5e637db095795bf6d62f24052a6677d661eab26b245f0c4bcb641d32aed5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"0ebf587cc6459eef731963fa3be03fe33cde248d08969e6a70b3dff755499dc1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"0fd10f7adb318f04cc70acab6f070e301c962c76c0f2e87c60a5b3a497bfb7e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"127700ec5e33f26ce5d03a644162639018e69956a32467f2b6b4f96981bf5e63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"138810299a5d011c12292b9abb117ec0ecdfa81de0d4e4de3f7c9e8a4373e91c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"225327a1c396b0769bbc3bf50ef512c3da85746f9b2a8f964c39db2cdf71fbb3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"22fd7ff12b38866f6bea49197d9440aed4e9f536744dd7a4eb72fe4c728dfe01","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"3494e7599369e84d2a659d76a6fb8d802d9a72358dae309791c85f491bd173ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"37c937391e28bd146b46c013902e04314f8df75e2f0f6b58e86b90e21294f2dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"3cde2c9351aa0baa177f95f6ccffaee52cfed5e5cdbb3454624f01ec75aab444","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"4befb7ee650d6ad554aa7e94d72b9ccaa413503f8e6994c899b5444f355fbf9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"4d7def40815be06bc421f84b58be402af0501dfd43bb41bff06943392b652441","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"69ca1d7a98769989b9cb8154b609a28a84f0993b6f599f5dcf4310d72fd0960d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"6f5895b1d00c4c1bd91fc6380332c7cca5422696069e4340c9bd6cea67de3620","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"71df655dc8b3df8dd84914dba77cbfd9af30ef3e92620e9c6c62491ed8848b75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"74d4f0ae3b4f2dc4ad72cc535df54a31cd637776bb7531edd21a20dca4d96c79","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"77f763b7ded0037b057501794ba9a529866e7a00ef7162714ce37403540159bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"7a48f0b30011a1bb3fba2e2afa094ca9638c7b995605f7cf986fcb955cf32a3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"7ccfe0f572b1d9e5d9d4b979b97e925d3e4f1abe5a116b0aa9ed1706bb328a42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"7d29e7a75ef4a9cbf8e36182951b1e18005c05cfede5344df8d95eeeb2e84b4a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"806361f013388af4076c1148af2638d35c07c964c6514a581fd2e1b8f880bc1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"825a4f6de44590f824ccae8b94feef8060135538d8a043cb6279b2740db84451","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"84aab62b96f3987b2a5c3c99763b33c22732dff552d3d9ed8ce120de6a2c4f88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"855cffaf58d7d6ca7e425993b0f03cc96471d3d483a8ed6b997f9fd222a0abe0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"856a1ca739e5a4f811ff9e09e7a13277a55898b7417ef6ec7304db07a70659e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"8b64b801dbad869c2789f0c647796a70435d37178f46fa6ede2039ab0bd89cf9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"8d5409c45fa53b4f0773b8132c5f222558c80e211ed9ac63d3921fd1daaa31d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"971119c8f7c24b90382591a57e1f74073a05400ada6da296237fddfbacb22b81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"aa26510003a45942f2895f18573abe057bc51f260e417f0ec5984104d779d1b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"af6e66e7c240bd0b73e520cb2865b4a0ced127bb6b0a56af430b3d481be0687c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"afa84f08cf2ece39444ff48d360f04a5db91e930865062e84f1023a9623d02b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"b092d8520dc47afae80010722758eb249da4d5da14bfd421bea99e5f2b43d1d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"baa7ca33b7ead40f99720a5619f789bf9042d54158df43cae9aec5865bbe1198","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"c2e5a4d2b5a42b030371cb68b478ab9c86f8c733bd2a7fa95dfdd1f1be8f62da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"c9cf9b84e3767cfc2d81efdb73609c2494d6f056800eef420d997433022f49af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"d5f64c646be402fec124647e3bfb30a775e8f23293201e2e9961872224f499b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"d771788e8cbe4e2f0278b2c212aad176ea3daf26dd727c85b11500d32d107648","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"dc2434063eeac130a52e7b92205112e288c88d0cbe7451594f5fb8c380b4b44d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"dce04364c9af0317884e21fc46c8cc348f1a89bb5ac4f731650116693bffc56d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"e69c1f84b6ca09a194571d37109666313a976b378e50249aeb26c1f2bc89fe16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"f8c5c10b57c93a0db1500e28e2f580f5bb6abe6098d144688463b9d62af9176b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-7q54w4, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7q54w4\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":43,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":43,"after_last_push_count":78,"trend_counts":[0,0,0,0,4,22,43],"trend_range":"2026-07-12..2026-07-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-15","2026-06-22","2026-07-06"],"prior_runs_affected":27,"prior_last_seen_at":"2026-07-10T04:27:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:754]: failed to create nodepool npdg-421-419\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001631260\u003e: \n    failed to create NodePool npdg-421-419, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npdg-421-419 in resource group rg-np-dg-minor-x5pk85-r87jrl, error: failed waiting for nodepool=\"npdg-421-419\" for cluster \"np-dg-minor-x5pk85\" in resourcegroup=\"rg-np-dg-minor-x5pk85-r87jrl\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:754]: failed to create nodepool npdg-421-419\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001631260\u003e: \n    failed to create NodePool npdg-421-419, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npdg-421-419 in resource group rg-np-dg-minor-x5pk85-r87jrl, error: failed waiting for nodepool=\"npdg-421-419\" for cluster \"np-dg-minor-x5pk85\" in resourcegroup=\"rg-np-dg-minor-x5pk85-r87jrl\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:151]: failed to create node pool npupgrade-4-20 with version 4.20.29\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011856e0\u003e: \n    failed to create NodePool npupgrade-4-20, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-20 in resource group rg-np-version-upgrade-nb5xvq-rqzwb6, error: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-nb5xvq\" in resourcegroup=\"rg-np-version-upgrade-nb5xvq-rqzwb6\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:151]: failed to create node pool npupgrade-4-20 with version 4.20.29\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011856e0\u003e: \n    failed to create NodePool npupgrade-4-20, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-20 in resource group rg-np-version-upgrade-nb5xvq-rqzwb6, error: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-nb5xvq\" in resourcegroup=\"rg-np-version-upgrade-nb5xvq-rqzwb6\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:95]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00146f290\u003e: \n    failed to create NodePool nodepool-128GiB, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-tmfkgt, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-tmfkgt\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:95]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00146f290\u003e: \n    failed to create NodePool nodepool-128GiB, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-tmfkgt, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-tmfkgt\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"027ac8898572bc71526fd0b49ec619428168e5e09712da6d06f0d277d76c61f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"15a71ab61dcbe9eaad0ef86ac019f0815c20d75aa244394e3beaf72279d9609d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"1e8e17719a82938ad90ae57e97fc88a0d150dfaa0c73b399681f224e680ce168","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"26b701ae4480ff79019e026c731df9ae386a5557b6c99f7ffffd963990cd7e20","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"296fc759c48def909625eec86bee51fc5fb12c39d87d7bdadc04d1b4df21a3df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"2cb682de538b2f1efea7e2bfc26cce0fd43b882f7fd84d0ec416524142621813","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"3eb37e36320fb739164e982e3b837ea950135aa6cb8d07f16e7e9dc9adb1d03d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"417136631f861c2335086adc813fca08fbabfda3f0e847f6372d3cf7237304e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"51a9aef95590007e266dd5d32148d556485e53a29fdda42e1ec43213dd06665f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"5833e1f5fc5f0f6d6b524e6ec32562462e6087e69daaba6d85f508ea1b53b2b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"65ee54afc06735cd79152b049519586f5a623a7f07f2b24e3e8faa277609a952","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"6bc2cc91b86eaa2f0bd23f69c74174de995d4979d4c0a8a1899840bc7495e614","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"8050aa75ea5a82051ffad94a969745f4e2e0dbec144fd9b4d283dd57fbbf8a0a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"8705200461c1095f0026d8e5006762389b5f8be898d3c1690f7320b48c9d6d37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"8b5acd0654c60184501a665b3667998fe95b15fa716defe91e12bb513166fa82","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"aab5bf5cfa436fda1ce894617fa3370db5ce7bcff09c51557618690432420ef4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"b019b3dc05f91d513edd26d51e30dd99aeeafaca7f8b70480b3306d726a187cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"b38e0d2f46dff5eee11de2a7f8c15052f681c436a8f7965482cde8c937657ac2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"b4552cc4f4ec017d1345c40732e121f50d08f351901ebe82b28fabc5cd2e5a03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"d0e407ef9fe96df97348ec4a91b5132246634fe1d2b06f9ca03d918c7a4f523e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"d5e9f8e10af8acb59b3223a01765d6b629a7209922c32d918e5de384877c463e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"d8678403d82e0ae68e52ac73143dbe1d9312d4fa56c96b79ccee6623f62ef3b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"e35be49d1355e2c6109ef3ef579ed71be7f257e08a3e48870ac264f6e152ccf3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"ee7bbc776f7b832726dd132f72bfd2fc9aea0acf1fa6ee16a11ffb98207d57c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"0465edc59652aae0a49da0d9fc5878e067da3991a8366f6ff1dbdda554912d13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"0999e665f69f1af62c4841d2d3b390877845e621c130de1ea0dc4ef1def9e53a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"0e4d8e2db72b48010453a3d1dc401f23fe8aa99736d18073cfc347a8503323b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"2c2a0d956cc853277844b978315542742bcd92ef8ccf9efce226232176223516","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"662685ac3f0f537db1c38aeda5bd2cfb4f293fc7186ef36b421a52b48266b33d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"87c05b14d0bbc6896fe7e15cebbc7717d47b96c189a7d96709d82a45a5868cca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"8931bab125f172b91101964a6cae48c20fbf5b2b20aa95332d5bc6215c346c94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"8e1f711d9f2d62b90f986e7ef4dd28f1c75f1201d017b814edf41018e9b665d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"9feffd6618cc1510790d28bd6fef588a08954e974766a468d54028d0df9a10ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"a3780701a4d7e0f63a1baa2378f6b28ef68396b31c4c65dd33f5adcd341e855a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"bbff5e1e82807a88ee6b3cc021857414782d1b328f1f75008fba94cc3a274b92","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"bffc0e8794fb1e14b00be896a24cccb1d979f468225893be791839584591b77e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"c3eec97c1b485d42e539759cc0efaa33e4c00df6173bfef792571c411a5f8a21","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"c940f4f6d0aba3e734f2020a57e9d8c0ae350f573b9110286ff1baa942eecd2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"cec1b3db85e3c8818cd4a6da13185f14baf1d8860a34f81d79bf769bb39c0403","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"ddc3e6bf23b734d6df45d48034ecfb412d849724cb899b6f51b868c3add48c6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"f10c5e5e7803593a9efcfa570360e71c61e911025e6ee0b7474ae55a0c79260f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"f4a36aafa49a0b7b9ee1fd78a6d40173fcfb413ae4ff5e500ddf4662cfaf472e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"fa6d1c1dac2d43f021418bf17509d12a7a1ae838fd269912cb9c594b54b132e6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-klvg94\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-klvg94, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":19,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":19,"after_last_push_count":19,"trend_counts":[0,0,0,0,0,0,19],"trend_range":"2026-07-12..2026-07-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-15","2026-06-22","2026-06-29","2026-07-06"],"prior_runs_affected":61,"prior_last_seen_at":"2026-07-11T23:00:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001366b70\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-zmr4rc, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-zmr4rc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-zmr4rc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001366b70\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-zmr4rc, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-zmr4rc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-zmr4rc, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-jfnmhv\" with version 4.19.37 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ecb920\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-jfnmhv, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-jfnmhv in resource group rg-zstream-upgrade-4-19-pqtf6j, error: failed waiting for cluster=\"cluster-zstream-4-19-jfnmhv\" in resourcegroup=\"rg-zstream-upgrade-4-19-pqtf6j\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-jfnmhv in resource group rg-zstream-upgrade-4-19-pqtf6j, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-jfnmhv\" with version 4.19.37 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ecb920\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-jfnmhv, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-jfnmhv in resource group rg-zstream-upgrade-4-19-pqtf6j, error: failed waiting for cluster=\"cluster-zstream-4-19-jfnmhv\" in resourcegroup=\"rg-zstream-upgrade-4-19-pqtf6j\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-jfnmhv in resource group rg-zstream-upgrade-4-19-pqtf6j, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a66000\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-26zxns\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-26zxns, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a66000\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-26zxns\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-26zxns, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"064e7b89f9ff2e844d13cd06007bbce7026bd03910357ddcbcca551ad4a79ae7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"0dc886c39e1cd24386ba4e1bb784eafd3db519863e72b5ec0472ba57351f8871","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"21ef3a62444bca3442ccac69a57b3495ff80ae8c54dc2e727767d45267cbe939","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"352618344fe8f558600e2ecdc67597a394b0cfd69898f471d7711ddcc4502fc2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"3f64e4d4456bf8873f693a4ab423a709b436bb874f6f6acaf0515f652e874ecf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"4cf9757b09c215098ca095ba8c6ab80e18ac33ab501c20d9f6cb261ae6c0c82d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"5128c2a59199c49431370e557412a3aaae1c58b1c88c68c974e7fd00f685a51d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"f57b9de3c41606031895a3055cd3102b12d4f6697934c4b996de5dac294db66d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"01d001baff3728bf155a18c5a5e6191b94055fa7fd0d3cdbc597162001dd7d12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"025d5a84f2741477ddc0e49ae71a81223220afaaf5c66566f313f2ee040208d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"4184c7f8e2ac3e4f2aed2092f549793729e34e558388a2b74ca42b5bfdcedb8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"4c51b46ecf041e9a164d7dbf70de2925cd64474c0538d3d105e00d66bf1eec9b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"640d4726cb7fce2a7df4f22671306ea05d6ef6183c3709a61b571dcdde925cfd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"b8e99d53d5d1eeeeed14600e89b174f40e3ca606e96a412d6ab61745a8de0a1b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"be6e8e3722a341b1336da01f0c7e716de6e500b136d4ccaaeb6ef72c0a5c3cc7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"d06442182c4f04f39e0c1a80ddd0b859740e947695cd9bec3269947c753bf109","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"db0a42f0727044c775aca72254e6b855f04622261ef1a3bd85d96022014d8b73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"e637183742d30ad7547b4597ae3ad6831de3d046ab891b9402a0360883d96338","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"f9b94b0b2a5f3c81cfae02be60567332260c197c6644f992bb066e3d1d28026a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-jtqshr\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":75,"anchor_occurrences":3,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,1,3],"trend_range":"2026-07-12..2026-07-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-15","2026-07-06"],"prior_runs_affected":7,"prior_last_seen_at":"2026-07-08T09:41:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9ea80\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-n44gmr\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9ea80\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-n44gmr\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:140]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000849c60\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-jtqshr\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:140]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000849c60\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-jtqshr\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00298b220\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-f2grmp\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00298b220\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-f2grmp\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"8b8bdbe1e43422af30a09e72b3bb64eec4a391f1c6a3d2ac99f34db40ff9ba5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"6aa83e909fc9b93220981451236e05304b21d1389a520be6a3053e911d2ec004","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"3a2f42ef566f34024f4639034a2778fe07470d857a1984dc044c17d9bc8c4b80","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f1386f529db6e94f57caba5baeea25a7ed546b620757afe7e7438f6ec564ee32","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-ds4kvv\" for cluster=\"ea-list2lxps8\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list2lxps8 in resource group ea-list-ds4kvv, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,2],"trend_range":"2026-07-12..2026-07-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:120]: failed to create external auth config on cluster ea-listwf222v\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010e0510\u003e: \n    failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-j9xw65\" for cluster=\"ea-listwf222v\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listwf222v in resource group ea-list-j9xw65, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:120]: failed to create external auth config on cluster ea-listwf222v\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010e0510\u003e: \n    failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-j9xw65\" for cluster=\"ea-listwf222v\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listwf222v in resource group ea-list-j9xw65, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:120]: failed to create external auth config on cluster ea-list2lxps8\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011a2240\u003e: \n    failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-ds4kvv\" for cluster=\"ea-list2lxps8\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list2lxps8 in resource group ea-list-ds4kvv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:120]: failed to create external auth config on cluster ea-list2lxps8\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011a2240\u003e: \n    failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-ds4kvv\" for cluster=\"ea-list2lxps8\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list2lxps8 in resource group ea-list-ds4kvv, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078506349536219136","occurred_at":"2026-07-18T15:45:02Z","signature_id":"5a5ea97574c2d536dd9323078b70e22b95248cf4155ae96d892cfd199791aeb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"c7ec9dd9338e5a135dcc8c4f1df61b9cdf7bbc2d4908076527ca1e406acb3b9b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7ac9ea1609c23f13fc59d25bc0226d6ddbb91e9217b1f67c5624f53eb77e8333","failure_pattern":"error should indicate FIPS value must be true or false","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":17,"trend_counts":[0,0,0,1,2,3,2],"trend_range":"2026-07-12..2026-07-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-29","2026-07-06"],"prior_runs_affected":17,"prior_last_seen_at":"2026-07-08T02:00:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078615808023465984","occurred_at":"2026-07-18T23:00:02Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078298716225146880","occurred_at":"2026-07-18T02:00:01Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-12..2026-07-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-15","2026-06-22","2026-07-06"],"prior_runs_affected":11,"prior_last_seen_at":"2026-07-09T11:27:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001b6a0d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-7wkjqv, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7wkjqv\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001b6a0d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-7wkjqv, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7wkjqv\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078392721382313984","occurred_at":"2026-07-18T08:13:34Z","signature_id":"33a271358e9929722d2a055c01aee6944b0786984fef9265bca9539d89aa2c85","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}