{"meta":{"start_date":"2026-07-19","end_date":"2026-07-19","start_at":"2026-07-19T00:00:00Z","end_at":"2026-07-20T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T04:29:39Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":5,"failed_runs":5,"raw_occurrences":53,"matched_occurrences":53,"runs_affected":5},"rows":[{"environment":"prod","failure_pattern_id":"89cb7e06e525cd27136f4e938bd6f1470cdb9e65f768b9fa227d139fb3154e59","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-creation-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":43,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":43,"after_last_push_count":371,"trend_counts":[0,0,39,82,125,82,43],"trend_range":"2026-07-13..2026-07-19 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ccd860\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-m894z8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-m894z8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ccd860\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-m894z8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-m894z8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-wmldcc/cluster-candidate-4-21-ktkqmd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001281320\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-ktkqmd: failed starting cluster creation \"cluster-candidate-4-21-ktkqmd\" in resourcegroup=\"rg-candidate-4-21-wmldcc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-wmldcc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-ktkqmd\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-wmldcc/cluster-candidate-4-21-ktkqmd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001281320\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-ktkqmd: failed starting cluster creation \"cluster-candidate-4-21-ktkqmd\" in resourcegroup=\"rg-candidate-4-21-wmldcc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-wmldcc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-ktkqmd\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001247000\u003e: \n    failed to create HCP cluster np-dg-minor-nqpdmm: failed starting cluster creation \"np-dg-minor-nqpdmm\" in resourcegroup=\"rg-np-dg-minor-nqpdmm-jq94dp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-nqpdmm-jq94dp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-nqpdmm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001247000\u003e: \n    failed to create HCP cluster np-dg-minor-nqpdmm: failed starting cluster creation \"np-dg-minor-nqpdmm\" in resourcegroup=\"rg-np-dg-minor-nqpdmm-jq94dp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-nqpdmm-jq94dp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-nqpdmm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-creation-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-creation-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"04c1d4b03092cfc94687c66d708a61d41ef1e12623f6da26916b9262d54c8719","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"07c746d26ce80f48134509da4dd9eda844fe020045932f1b4c670fffd125cdc6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"080234c1500f80252f05eef038e49bc6d4e7e0576930864ef67722c24cfda29d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"11a17376da216ca74f4ee3b453a7e926815287d4bda0ebb6eec90e2108060a1c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"17e0dbc2052db5b66f601f462532fbda679ce9aea103e62cff4fd74f9d83953e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"1af110b5e4d0aeaa52aafccd8bc774e4c2ce01f26fdce89039b1f8d9576f25e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"1b997f926c39d93671beaa3dc96e745d7a81e75fe5e8eeeddacf6793b97cf3b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"1f8b335859a2f9010cc5fe23c1ba5e4becf58c7977aa942bcd93943d0f7d59cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"25d6b7ac5bd9866377334238ff66a6af08b6279631b9dd544ad77c7a479e38db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"27a75c866b2b422396e29264dac00507f1b5e4eb61e45fc9c478a6c1c7fff75c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"28da511449e9c1995df7f8e59e7b0db6a81b8a07a9f0ceaec52cffaf24331962","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"3170911960bfc7a26370736bae1ac5b36343651df97cb6e6fd78c2b8bef6c3b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"3a3c83ac682e2fbdf2bd3be5c0478f7dc29a4f63b03a14ee67ca26c5b221e841","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"47469ba02a2a1c24c4a27ec9d5a45f57f799e1518020d9fc818e8520affd4b7a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"524144bdf72f934b6a687ae27294293199597b1bdf66374f81de7fe42125236e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"548d753067fc8e2b9eb930ef8630e7794df89a8059f931ad210d2dc18634c27e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"66eb1cc9830c133c736060c618459f7a549ecf3233ee32475eb644ba53eaa663","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"746735115dfa2aecfd2341194846216ce1669ef92f7557d6a3a118343e948db0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"7554d66ad05c6c33a496927599c60cc86b191df9cbc0a54aab5fdb201c362729","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"75de912ba7409bcfa3827d6b53086f0a5b872f8e3468d5238d941c732257b93b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"7eff81be787f07020c0519b62f2ee32781d311ab3da1552744f16b734e030946","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"8821e005a4d988916aafaeec809a732f08dcd8340ba8b67d91c233b2e22dda14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"8d38069ad3ee3f66f6168e33021eb3f31350d24434bde186f0d67f78c38bdddb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"8d42936f13fd94bff5ac69582e89ef52531656facae1825b478a771ca72da00d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"90367a69b1b130b5ea4e0672ebfd06f4954c5c55126c57476d166ea9654924f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"91298696362030e2434e0c190add8be24c6bdf797279effb10bbeaa1c9ccb2d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"959949469c5599cccad1bd28f838e643dbdaf7e9206d5ca7c0a8da30448f95c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"9dd73bfd0ff0bfbee30d2350c40591685bb50ee9ae788c1a3451f1f106440b63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"a51ed0c84dfa7f12dd5b03d2b8f861aee9b6a0eb53d9d72b27b1fffc95540035","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"aa5b4bd84d9be77dcc220207dfd7386104d84659adf95ce944dfa5dac3adeb87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"ab271e24e1757fc5cf12a4fd6b8cb5f9cbb8104dc83693ab94431e12ff16171a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"aff3d74ddd77a59767a6e6bb0e658fff99a79f72fd9d4d2005fef9460825e5ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"b0bd4a075028b5800bea00559a10b1ba546b844a720279413cfc9979b82ab6e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"b1d706cadd9c6f5282a0745e0783348fbfe00e05964fbd56257bc18eb2d22c94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"b51bb12c4d01e223b059ea219bc22d349fb4a1b6f0272b6420bf7cc239918d43","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"b86a9f1db53db7b9aa61e1c1057d48d585e44990750d2d418d93d4a6f8d38551","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"ba3b89a970c9cc3856fbecb01deb4bad9ff18af6d01f82965fe0f6eebeddee96","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"c509155ae10f8bbb21b054361725d4019448edbab396f550e28364e61110e2d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"cdd492c195f2d6d234fc3ecaa5eb22d5487fcbb3b4db4efef350c2269a9b6c9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"dd84c1419a9b479b494e4a332bba7f4f7b26094e5cf9f4b7f9949704fce2c9e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"eafc592df1b01304dc40753bb69fb681cdd3cc90f7899ea551dd87c13eaf9c4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"fce3b2279646eb3bbda794f8289a5c25eb28ac49c6e963360d9ce5b8b9ea4b7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"fe82a8e71009f22439176335465e43b123ed6bc3085b8f3cf333fdb73ecd3318","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-2k9w77, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-2k9w77 in resource group rg-candidate-5-0-7c98j7, error: failed waiting for cluster=\"cluster-candidate-5-0-2k9w77\" in resourcegroup=\"rg-candidate-5-0-7c98j7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-2k9w77 in resource group rg-candidate-5-0-7c98j7, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":80,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-07-13..2026-07-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-15","2026-06-22","2026-06-29","2026-07-06"],"prior_runs_affected":65,"prior_last_seen_at":"2026-07-11T23:00:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-rcpcj4/cluster-candidate-5-0-qh92sz should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001883b30\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-qh92sz, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-qh92sz in resource group rg-candidate-5-0-rcpcj4, error: failed waiting for cluster=\"cluster-candidate-5-0-qh92sz\" in resourcegroup=\"rg-candidate-5-0-rcpcj4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-qh92sz in resource group rg-candidate-5-0-rcpcj4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-rcpcj4/cluster-candidate-5-0-qh92sz should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001883b30\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-qh92sz, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-qh92sz in resource group rg-candidate-5-0-rcpcj4, error: failed waiting for cluster=\"cluster-candidate-5-0-qh92sz\" in resourcegroup=\"rg-candidate-5-0-rcpcj4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-qh92sz in resource group rg-candidate-5-0-rcpcj4, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-qkq52s/cluster-nightly-5-0-x5kc9l should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000318f30\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-x5kc9l, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-x5kc9l in resource group rg-nightly-5-0-qkq52s, error: failed waiting for cluster=\"cluster-nightly-5-0-x5kc9l\" in resourcegroup=\"rg-nightly-5-0-qkq52s\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-x5kc9l in resource group rg-nightly-5-0-qkq52s, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-qkq52s/cluster-nightly-5-0-x5kc9l should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000318f30\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-x5kc9l, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-x5kc9l in resource group rg-nightly-5-0-qkq52s, error: failed waiting for cluster=\"cluster-nightly-5-0-x5kc9l\" in resourcegroup=\"rg-nightly-5-0-qkq52s\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-x5kc9l in resource group rg-nightly-5-0-qkq52s, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-23-gvmcfn/cluster-nightly-4-23-lpfl26 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001344660\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-lpfl26, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-lpfl26 in resource group rg-nightly-4-23-gvmcfn, error: failed waiting for cluster=\"cluster-nightly-4-23-lpfl26\" in resourcegroup=\"rg-nightly-4-23-gvmcfn\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-lpfl26 in resource group rg-nightly-4-23-gvmcfn, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-23-gvmcfn/cluster-nightly-4-23-lpfl26 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001344660\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-lpfl26, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-lpfl26 in resource group rg-nightly-4-23-gvmcfn, error: failed waiting for cluster=\"cluster-nightly-4-23-lpfl26\" in resourcegroup=\"rg-nightly-4-23-gvmcfn\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-lpfl26 in resource group rg-nightly-4-23-gvmcfn, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078982133782679552","occurred_at":"2026-07-19T23:15:40Z","signature_id":"efc8c54a7baff8abe06d202310a3acd723fcc8ffee15a6d351c85a443ee9a1ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078978406745116672","occurred_at":"2026-07-19T23:00:52Z","signature_id":"6d4c740a3954d6cb7f587ef2d7ffb63fecf4b2a905a654be81c1b7fac87ca3f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078978406745116672","occurred_at":"2026-07-19T23:00:52Z","signature_id":"a3ac5958e1b716b3d65f008902d905620eca942dd2f70b9642e8b879f3076105","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078888703945609216","occurred_at":"2026-07-19T17:04:25Z","signature_id":"cd91c340e934fab0578f7209d7d8be317d36dba9b9c7faf4227fd2e75fb3a2cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078837200895610880","occurred_at":"2026-07-19T13:39:46Z","signature_id":"629adadd8a1d57fb3862e4237bb36617565afd28e8de2420d83611868d6c4d31","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"e2e-cidr-connectivity-q69csh\", caused by: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-q69csh\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-q69csh, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-q69csh\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-q69csh, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-13..2026-07-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-06"],"prior_runs_affected":7,"prior_last_seen_at":"2026-07-10T23:00:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017963c0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"e2e-cidr-connectivity-q69csh\", caused by: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-q69csh\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-q69csh, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-q69csh\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-q69csh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017963c0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"e2e-cidr-connectivity-q69csh\", caused by: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-q69csh\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-q69csh, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-q69csh\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-q69csh, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2078978406745116672","occurred_at":"2026-07-19T23:00:52Z","signature_id":"59412b955c263c629635e3c6539780ac6c6b941c38c547267ca819b276a4448d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-13..2026-07-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:100]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002817540\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-nldlnv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/49f7f654-188b-4b7f-83a4-dec06325f989\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/49f7f654-188b-4b7f-83a4-dec06325f989\",\n      \"name\": \"49f7f654-188b-4b7f-83a4-dec06325f989\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-19T23:56:32.405067105Z\",\n      \"endTime\": \"2026-07-20T00:15:34.763165445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:100]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002817540\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-nldlnv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/49f7f654-188b-4b7f-83a4-dec06325f989\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/49f7f654-188b-4b7f-83a4-dec06325f989\",\n      \"name\": \"49f7f654-188b-4b7f-83a4-dec06325f989\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-19T23:56:32.405067105Z\",\n      \"endTime\": \"2026-07-20T00:15:34.763165445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078982133782679552","occurred_at":"2026-07-19T23:15:40Z","signature_id":"78733920d32a9b7d9c6e732c2596ca203360e8ee9114e9e2a1a1f017e0dd946d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"63a755b10a4ebc8c89991a40e36c8d3c0a9cd5e114a858de40038cb6cec08053","failure_pattern":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-13..2026-07-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-15"],"prior_runs_affected":3,"prior_last_seen_at":"2026-06-21T02:26:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:269]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a07bf0\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:269]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a07bf0\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2078888703945609216","occurred_at":"2026-07-19T17:04:25Z","signature_id":"607f6cb474ca772b1abc5ee7e2b356fe0eb411ef92627aab8b659dfa8b32576f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7ac9ea1609c23f13fc59d25bc0226d6ddbb91e9217b1f67c5624f53eb77e8333","failure_pattern":"error should indicate FIPS value must be true or false","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":18,"trend_counts":[0,0,1,2,3,2,1],"trend_range":"2026-07-13..2026-07-19 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-29","2026-07-06"],"prior_runs_affected":18,"prior_last_seen_at":"2026-07-08T02:00:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-zzf77k\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-13..2026-07-19 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-15","2026-07-06"],"prior_runs_affected":8,"prior_last_seen_at":"2026-07-08T09:41:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d4a660\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-zzf77k\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d4a660\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-zzf77k\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2078661107135811584","occurred_at":"2026-07-19T02:00:02Z","signature_id":"d3a0e302fd4ca1ffa0c16f2b12844335acc4ae17ae275eba1c35c4d30b0c00ca","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}