{"meta":{"start_date":"2026-08-05","end_date":"2026-08-05","start_at":"2026-08-05T00:00:00Z","end_at":"2026-08-06T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T05:19:47Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":15,"failed_runs":6,"raw_occurrences":62,"matched_occurrences":62,"runs_affected":6},"rows":[{"environment":"prod","failure_pattern_id":"4035e44f5e2eda00878b0b1b6c1eadc2bdf56ef52966cc244b842b956c95ea0d","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-deletion-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":44,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":44,"after_last_push_count":127,"trend_counts":[0,0,0,0,0,83,44],"trend_range":"2026-07-30..2026-08-05 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be9f40\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-zgrvhfc5n22r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-zgrvhfc5n22r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be9f40\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-zgrvhfc5n22r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-zgrvhfc5n22r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-20-sq4lngxskx9k/cluster-candidate-4-20-ktfxxt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e76960\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-ktfxxt: failed starting cluster creation \"cluster-candidate-4-20-ktfxxt\" in resourcegroup=\"rg-candidate-4-20-sq4lngxskx9k\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-sq4lngxskx9k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-ktfxxt\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-20-sq4lngxskx9k/cluster-candidate-4-20-ktfxxt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e76960\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-ktfxxt: failed starting cluster creation \"cluster-candidate-4-20-ktfxxt\" in resourcegroup=\"rg-candidate-4-20-sq4lngxskx9k\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-sq4lngxskx9k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-ktfxxt\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-21-f69fr7gsvs5v/cluster-candidate-4-21-7bfkvq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b83160\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-7bfkvq: failed starting cluster creation \"cluster-candidate-4-21-7bfkvq\" in resourcegroup=\"rg-candidate-4-21-f69fr7gsvs5v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-f69fr7gsvs5v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-7bfkvq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-21-f69fr7gsvs5v/cluster-candidate-4-21-7bfkvq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b83160\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-7bfkvq: failed starting cluster creation \"cluster-candidate-4-21-7bfkvq\" in resourcegroup=\"rg-candidate-4-21-f69fr7gsvs5v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-f69fr7gsvs5v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-7bfkvq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-deletion-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-deletion-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"03ca5588a6d5ccfca9529ca352fe15e131dc14fe6b97fc8eaa31b68af0d37fb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"03d9f2660879ea7816bc81e205b8dd4619bb6aea8225c4136453f47f50eca4d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"09de15d9624444e93e6bf987dffc833cd53e39c16a86437e97fd60913d644250","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"10de3f5d12e187c4ba880683b77390baca238e257952271f22d35c2870cb12e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"151bb421df6c35de4d71a921fa0652ef93b9d4b444d74511256c55bf88999c77","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"1c31c6bc3f9173f506da8b1b1183e76cd9b1b75187d50dabe6dc50fd5555aa71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"1e158c248c7895a2968fe0b4467091c42c9b8d3853a0e1d3b43fa15e2008d959","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"23e5f822309cc1e9c01f2db4de7fae3c72742de7f45518582492346bc4edd84b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"2820100c541c9229944937942cb8d787d42dd0871517a5115ff54a877d664822","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"318ea146134173a78f55b8d68c490958f9709535bde129bca8cba71dbd3b430e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"3b54516e07b66402eb815a7135d72999c0f847f54d60d7268cab74840aaf2e2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"456c3166931597502adeb28ddcddd2fc60b544fea344bd8285f605481c6f19e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"48c508e42520522a03b7c669563534004de1b261e43e421ddcf62557809cc962","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"4abbc1a05acca2f17a270bcda87e4f3edee9ecd69c41d9296dacdbc268ee8b6b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"4eb8cf5247e1175624ebc586f4a9ceadab8907b430637a2022df29df86cea6c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"55172ed61dca08ebe3b1b0ef1b612788bf3981ef588c9b9c7b78d353b2dc8d58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"5b18f47b720381408ae0a10d5aa276cc06ca15f1c1e3615f528f23deaf94c488","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"613639b34c0ba2c7fa43b538fa0a165cb1d21c6c86f7b5482797565d2c8352a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"6cd81fc9f402c323bfb4e4df5ebc0a57b6261e522a6c966e6b69342262e84373","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"722f43dd6402a452b50618297d9730370ac258282b7694ba0b75b9aeef5313e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"7e9066c154a432fa31bf0f54661eb7f706f23595b21dc7038bd24a3a5a5a2bf4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"888e42ea7a2f0fd3efb3efdcc0eed37ffce6bd2fd21808f1e88f8e5279b10eb8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"8c44d492c1a869c6423cd4cf609342a5c1acfe40f18a99411fa912e738a2ac02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"9542edd37b4734b7d505b5d73cd202208c55322a80121ce628b74fe4f974adb1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"9c9f18d5ef1ecdcb9b9cc30cdf1b197e5b0df76bb0938ee6cb8d0d2bfb66df11","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"9cbafb69ec331ca9dacf8b78fd024215f761d05d2023a155eb397ea8dc073424","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"a103841e1f920539b9907f65dd4a97e83c4feb8dc319d414cd89508f45697eb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"a1b7fcdc4526d3eed3bb2cf0e4f93d318cee6f6dfb7b5434b250c5d801f9b97d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"a359271faf30ecc2ff4d41db83d5780b53fd9ef4ea590ae481df44368e4a560c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"a8fec5104a40af90d4881cb81832d5fa132463243203001e06e18a58c952b4a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"b01aa812bb6a54b76c9f2192d7e183da3982cb9e52eadc2bd9490b0b906ae9e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"b0ac9daad8990dc03094a99b65d4ba485d8b6ff900ae5c3600e4293380b80460","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"b21afd9ddd5e9f5de4d6b0bcb4a6187759b9e17f75c2b678c0189819d833a1c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"b2a553e93b5a9584ee36e839f129be259641ae199e7c28b504e27cea02ca55fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"c21d772b22b9fb8ba77e87dbf4c3ea34266aae237b9a4b71622bfa8303b6172d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"c2a8f7be5df54c2520c94a9c655b2dd19b7c187e4bfef26bfb6e226a76d7b146","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"c5f3cfb7ed0681d486275ae9f03df21ec5a0168a6adbcfb1491ca5bfd171d7e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"c8ae398eef451111ba70a990fb1a9b871cd6c4e44a2c96275462d5389951af37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"c9ec6a53cd25e43a397a9d4056dc2aefe0b2eff4d58819b25725d596987dbd8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"cfd01877049d84840a5266149d64e2b9743dc659e4cc3a4a4f535d21d7dabd0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"d07f2da0734e9ddb73b555867ec875d02926f288b0620f9666d9c5461223778b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"ddce7ca7bbc12d124d0cbdd62dd6b20519e10df13bdf13de1e482dce46a207eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"de851742dce32fec2b63776b872ae3b81315b96c525bda9abb50f8f288dc8b71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2084821702679728128","occurred_at":"2026-08-05T02:00:02Z","signature_id":"f9fff26eeb6b461f3d4bf4c50cdf407684a6ab471fe29a567d90fe82be99b1eb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":12,"runs_affected":2,"failed_runs":2,"run_impact_percent":13.333333333333334,"anchor_occurrences":12,"after_last_push_count":115,"trend_counts":[4,47,0,29,0,12,12],"trend_range":"2026-07-30..2026-08-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-27"],"prior_runs_affected":15,"prior_last_seen_at":"2026-08-02T09:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9f080\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-595qvhvlzhs8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6f6cae5-88cb-4606-8acd-ff65a9d7e5ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6f6cae5-88cb-4606-8acd-ff65a9d7e5ae\",\n      \"name\": \"d6f6cae5-88cb-4606-8acd-ff65a9d7e5ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T23:02:02.959051037Z\",\n      \"endTime\": \"2026-08-05T23:21:06.707682725Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9f080\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-595qvhvlzhs8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6f6cae5-88cb-4606-8acd-ff65a9d7e5ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6f6cae5-88cb-4606-8acd-ff65a9d7e5ae\",\n      \"name\": \"d6f6cae5-88cb-4606-8acd-ff65a9d7e5ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T23:02:02.959051037Z\",\n      \"endTime\": \"2026-08-05T23:21:06.707682725Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d037a0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qzwfqjmv66dg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/88fc7561-3d60-456b-a68a-6ca923daa66a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/88fc7561-3d60-456b-a68a-6ca923daa66a\",\n      \"name\": \"88fc7561-3d60-456b-a68a-6ca923daa66a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T23:02:46.843171403Z\",\n      \"endTime\": \"2026-08-05T23:21:46.479374024Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d037a0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qzwfqjmv66dg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/88fc7561-3d60-456b-a68a-6ca923daa66a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/88fc7561-3d60-456b-a68a-6ca923daa66a\",\n      \"name\": \"88fc7561-3d60-456b-a68a-6ca923daa66a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T23:02:46.843171403Z\",\n      \"endTime\": \"2026-08-05T23:21:46.479374024Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f08f60\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gplz5tqmj4lz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fd15cdd1-c44c-4fce-a769-64adca7f0714\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fd15cdd1-c44c-4fce-a769-64adca7f0714\",\n      \"name\": \"fd15cdd1-c44c-4fce-a769-64adca7f0714\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T23:02:03.150425606Z\",\n      \"endTime\": \"2026-08-05T23:21:06.68148912Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f08f60\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gplz5tqmj4lz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fd15cdd1-c44c-4fce-a769-64adca7f0714\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fd15cdd1-c44c-4fce-a769-64adca7f0714\",\n      \"name\": \"fd15cdd1-c44c-4fce-a769-64adca7f0714\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T23:02:03.150425606Z\",\n      \"endTime\": \"2026-08-05T23:21:06.68148912Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"08598c702ff15f33afec0e5639dc73f776279c2738641d0d10b0824a07a05a29","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"2dd17e0d6d0b4aec1c387011f2065976166e2b25b25e613812b7d9ed8ea630c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"36ec2f0faefdd64ee3141448f5d7ea2bb4f7ca992a71361ad75d894f6ada2eb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"4556ea65211f0beb6cb5f0494fb1adfc7b76928ad5716a56e3ca1c9711370afd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"5461a3f3f95f6df7ae047d1d5621eeebb5562cc076ff4866e9fb27cfe652beec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"609f8f37aef29a3f82de90d62497d16c206e0bc090300c6407688069233fd07a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"89f055d074b543fdbd1f9269584a9f82284e86bd09ed144dbbeeaffcb4c30b3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"9226d2f2aa986e80238477b17e68fbc96e81f25d70bfaa783bbdd2a73cdca981","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"ac1ba843d5790f1712ba1c5f6f9dd81e26b8223bd2b53890d944195cda747975","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"b165651a3935c38e4cad6651ba04430382ce5597407df7ba93b839e61bcb687c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"c2df7b85900c9a7db7d86d3cfa1c61c534f326f91db67f2a1149f4cbd8ef1eac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085136126972006400","occurred_at":"2026-08-05T22:49:27Z","signature_id":"8a2200ed713f2a7af80f6b1e67f08f27206d0da45a7832e6169b2270ce7b206c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":13.333333333333334,"anchor_occurrences":2,"after_last_push_count":24,"trend_counts":[5,1,0,2,1,1,2],"trend_range":"2026-07-30..2026-08-05 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-13","2026-07-20","2026-07-27"],"prior_runs_affected":21,"prior_last_seen_at":"2026-08-02T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:260]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c82460\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-tnhxdw9k9dcw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7792344a-f40f-450c-8a0d-fa7a5148a643\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7792344a-f40f-450c-8a0d-fa7a5148a643\",\n      \"name\": \"7792344a-f40f-450c-8a0d-fa7a5148a643\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T21:51:52.985006933Z\",\n      \"endTime\": \"2026-08-05T22:10:53.226083392Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:260]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c82460\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-tnhxdw9k9dcw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7792344a-f40f-450c-8a0d-fa7a5148a643\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7792344a-f40f-450c-8a0d-fa7a5148a643\",\n      \"name\": \"7792344a-f40f-450c-8a0d-fa7a5148a643\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T21:51:52.985006933Z\",\n      \"endTime\": \"2026-08-05T22:10:53.226083392Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:171]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0024c8378\u003e: \n    VerifyNodesReady failed: there are not ready nodes: [cilium-cl-cilium-np-kflpw-mq662]\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-qv68x75xt8mm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/db5aee65-ef02-43ce-8305-1afd109ed896\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/db5aee65-ef02-43ce-8305-1afd109ed896\",\n      \"name\": \"db5aee65-ef02-43ce-8305-1afd109ed896\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T12:56:40.149256028Z\",\n      \"endTime\": \"2026-08-05T13:15:46.46639045Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:171]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0024c8378\u003e: \n    VerifyNodesReady failed: there are not ready nodes: [cilium-cl-cilium-np-kflpw-mq662]\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-qv68x75xt8mm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/db5aee65-ef02-43ce-8305-1afd109ed896\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/db5aee65-ef02-43ce-8305-1afd109ed896\",\n      \"name\": \"db5aee65-ef02-43ce-8305-1afd109ed896\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-05T12:56:40.149256028Z\",\n      \"endTime\": \"2026-08-05T13:15:46.46639045Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085099167385915392","occurred_at":"2026-08-05T20:22:35Z","signature_id":"e6cfd63ef72606f530614324528351264962e5d5d68bd93977952d6f44967019","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084975910284431360","occurred_at":"2026-08-05T12:07:16Z","signature_id":"9b85d0bd47f779fa46cfd7bc64622665a46f651236ae23d5f3ac14611ded8d9a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-5crnhz2mwd5v, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-5crnhz2mwd5v\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-5crnhz2mwd5v, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-07-30..2026-08-05 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20","2026-07-27"],"prior_runs_affected":35,"prior_last_seen_at":"2026-08-02T09:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:149]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0018e83c0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-5nqx5p2gvzr9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-5nqx5p2gvzr9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:149]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0018e83c0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-5nqx5p2gvzr9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-5nqx5p2gvzr9, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001c1cb70\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-5crnhz2mwd5v, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-5crnhz2mwd5v\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-5crnhz2mwd5v, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001c1cb70\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-5crnhz2mwd5v, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-5crnhz2mwd5v\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-5crnhz2mwd5v, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"7b6e9ec3c8f7a6ebd65015d4fc79091ff849f7f5770aff2cb7f75ded8aef079f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"ace94c99de58d994657790547c4d4456710fad10b9cc554190a334ba85d9a970","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":1,"after_last_push_count":21,"trend_counts":[0,3,0,1,0,5,1],"trend_range":"2026-07-30..2026-08-05 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-20","2026-07-27"],"prior_runs_affected":21,"prior_last_seen_at":"2026-08-02T09:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085138797804130304","occurred_at":"2026-08-05T23:00:03Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":1,"after_last_push_count":11,"trend_counts":[1,4,0,0,0,1,1],"trend_range":"2026-07-30..2026-08-05 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20","2026-07-27"],"prior_runs_affected":14,"prior_last_seen_at":"2026-07-31T15:09:20Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-6qxttk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e661e0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-46xsl.apps.aro.x2j7q6d3f2a2p2q.60fr.brazilsouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-6qxttk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e661e0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-46xsl.apps.aro.x2j7q6d3f2a2p2q.60fr.brazilsouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2084975606633598976","occurred_at":"2026-08-05T12:11:36Z","signature_id":"9b26a8767aa4c9a1d3489fb2faa203a4bc9fd6e3348e63909580d96a47070c2a","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}