{"meta":{"start_date":"2026-07-30","end_date":"2026-07-30","start_at":"2026-07-30T00:00:00Z","end_at":"2026-07-31T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T05:33:43Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":5,"failed_runs":4,"raw_occurrences":10,"matched_occurrences":10,"runs_affected":4},"rows":[{"environment":"int","failure_pattern_id":"20ec3a566a193fff5b870a1e0c66e1d6dd360c49a81eeabea9c1bf418b6cea27","failure_pattern":"failed to create service principal: create service principal: error status code received from the API","search_query":"failed to create service principal: create service principal: error status code received from the API","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":7,"runs_affected":4,"failed_runs":4,"run_impact_percent":80,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-07-24..2026-07-30 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f05de0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f05de0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00150b9e0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00150b9e0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0002199a0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0002199a0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082864512502861824","occurred_at":"2026-07-30T16:22:52Z","signature_id":"bbcae88f9bd91ac84d2736736be0545becfbc88b8b707d6da8d236b24b32dd87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082864512502861824","occurred_at":"2026-07-30T16:22:52Z","signature_id":"cd8b18d54e848b0be1589211b07fc6892a04128fa02e494cfb003ff1579e65a8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082786541406523392","occurred_at":"2026-07-30T11:13:02Z","signature_id":"012e939c24855e55dc8fa5b4098f2594876ba83b1d2d68b4f3627221f48072a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082716084711985152","occurred_at":"2026-07-30T06:33:04Z","signature_id":"b4c17116a87ea8f85d066a721153d42c1bb9516ffa7272bd7761fa98c087615c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082716084711985152","occurred_at":"2026-07-30T06:33:04Z","signature_id":"eae8d475e571df1465546026575d2e0761f70dc0f38df88deeff37477fbb5fe1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082652897543196672","occurred_at":"2026-07-30T02:21:59Z","signature_id":"0a3cf3c8d5f004008a913b200180f87ff947a97638ff74d6a21dc4bbb03c1711","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082652897543196672","occurred_at":"2026-07-30T02:21:59Z","signature_id":"643fe0e349d725ddcfc9592ab7dcb78ba575a26497b9272fc7396e36b3b3be9a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"f90cae3228fe72b2427515e7c51834eebdc3e73c0b1aa6bca2abedd53a2e7578","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"patch-tags-4tr4cgtrvmhr\", caused by: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4tr4cgtrvmhr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4tr4cgtrvmhr, error: context deadline exceeded, error: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4tr4cgtrvmhr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4tr4cgtrvmhr, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-24..2026-07-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":12,"prior_last_seen_at":"2026-07-22T16:58:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e98318\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"patch-tags-4tr4cgtrvmhr\", caused by: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4tr4cgtrvmhr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4tr4cgtrvmhr, error: context deadline exceeded, error: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4tr4cgtrvmhr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4tr4cgtrvmhr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e98318\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"patch-tags-4tr4cgtrvmhr\", caused by: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4tr4cgtrvmhr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4tr4cgtrvmhr, error: context deadline exceeded, error: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4tr4cgtrvmhr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4tr4cgtrvmhr, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082864512502861824","occurred_at":"2026-07-30T16:22:52Z","signature_id":"883557dcd01d4fa8153b4f4d225b74622548173fe2b3838b46dbdd0e41063882","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e0eca219c3110341e80c5a3f64ae984723abc1e1208ac11199b51c76f20de3e7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,0,1,0,1,1],"trend_range":"2026-07-24..2026-07-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-13","2026-07-20"],"prior_runs_affected":6,"prior_last_seen_at":"2026-07-23T01:42:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:131]: failed to create GPU nodepool gpu-np-1 with VM size Standard_NC4as_T4_v3\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f22760\u003e: \n    failed to create NodePool gpu-np-1: failed waiting for nodepool=\"gpu-np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-nv2l5l98hkzq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7b8561b-0d3d-47ac-a57b-83cd6af7d375\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7b8561b-0d3d-47ac-a57b-83cd6af7d375\",\n      \"name\": \"d7b8561b-0d3d-47ac-a57b-83cd6af7d375\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T07:33:49.233891501Z\",\n      \"endTime\": \"2026-07-30T07:53:06.946835167Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:131]: failed to create GPU nodepool gpu-np-1 with VM size Standard_NC4as_T4_v3\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f22760\u003e: \n    failed to create NodePool gpu-np-1: failed waiting for nodepool=\"gpu-np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-nv2l5l98hkzq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7b8561b-0d3d-47ac-a57b-83cd6af7d375\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7b8561b-0d3d-47ac-a57b-83cd6af7d375\",\n      \"name\": \"d7b8561b-0d3d-47ac-a57b-83cd6af7d375\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T07:33:49.233891501Z\",\n      \"endTime\": \"2026-07-30T07:53:06.946835167Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082716084711985152","occurred_at":"2026-07-30T06:33:04Z","signature_id":"882351e0e6f3c040bae99057f0e95608e1726fe27b85fa498008879f0b71c7f1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"ecba9c70d623dbfd01226f49890ba9397974dd3b8819dd0355ce6670accfbc55","failure_pattern":"expected \u003ccount\u003e ready (and schedulable) nodes, found \u003ccount\u003e","search_query":"expected 3 ready (and schedulable) nodes in node pool \"npupgrade-4-21\", found 4","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-24..2026-07-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:232]: failed to verify 3 ready and schedulable nodes for nodepool npupgrade-4-21 after upgrade\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002b03f00\u003e: \n    expected 3 ready (and schedulable) nodes in node pool \"npupgrade-4-21\", found 4\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:232]: failed to verify 3 ready and schedulable nodes for nodepool npupgrade-4-21 after upgrade\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002b03f00\u003e: \n    expected 3 ready (and schedulable) nodes in node pool \"npupgrade-4-21\", found 4\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082716084711985152","occurred_at":"2026-07-30T06:33:04Z","signature_id":"7007959012ccc8a940760507dc35c9df32589e0d43616950347a0bd0afe52668","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}