{"meta":{"start_date":"2026-07-24","end_date":"2026-07-24","start_at":"2026-07-24T00:00:00Z","end_at":"2026-07-25T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T20:18:04Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":2,"raw_occurrences":3,"matched_occurrences":3,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[1,0,2,0,0,0,1],"trend_range":"2026-07-18..2026-07-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-13"],"prior_runs_affected":3,"prior_last_seen_at":"2026-07-18T23:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:138]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004ef8c0\u003e: \n    failed to create NodePool np-2: failed waiting for nodepool=\"np-2\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-7pjp2fntgwhc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c569a933-5592-4cb9-951a-4e439ed8e930\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c569a933-5592-4cb9-951a-4e439ed8e930\",\n      \"name\": \"c569a933-5592-4cb9-951a-4e439ed8e930\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-24T02:16:26.672204644Z\",\n      \"endTime\": \"2026-07-24T02:35:27.643263493Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:138]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004ef8c0\u003e: \n    failed to create NodePool np-2: failed waiting for nodepool=\"np-2\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-7pjp2fntgwhc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c569a933-5592-4cb9-951a-4e439ed8e930\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c569a933-5592-4cb9-951a-4e439ed8e930\",\n      \"name\": \"c569a933-5592-4cb9-951a-4e439ed8e930\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-24T02:16:26.672204644Z\",\n      \"endTime\": \"2026-07-24T02:35:27.643263493Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2080473235987632128","occurred_at":"2026-07-24T02:00:47Z","signature_id":"d9d95b5e04d2638d898684563003edb95885592f2ef99fddfee432bdb213cde3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"2c1b6e2e9a0cffd0b5c8baae2a1873f1cdcec411d83a27ab1b9a7bc7d25a1dc2","failure_pattern":"failed to create cluster role binding for external auth \u003cexternal-auth\u003e VM","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-18..2026-07-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:328]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:328]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2080473235987632128","occurred_at":"2026-07-24T02:00:47Z","signature_id":"4f4c331b7bc2053d2469b256d4be2c082415cc0b7c4a8902668003f5d043798f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,1,0,0,0,0,1],"trend_range":"2026-07-18..2026-07-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-22","2026-07-06","2026-07-13"],"prior_runs_affected":9,"prior_last_seen_at":"2026-07-19T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:186]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-9p9q96\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e3e200\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-5s5tl.apps.aro.t4h5n3m8h9b6b1y.pmy9.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:186]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-9p9q96\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e3e200\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-5s5tl.apps.aro.t4h5n3m8h9b6b1y.pmy9.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080621562049859584","occurred_at":"2026-07-24T11:50:11Z","signature_id":"d4ce604404a11bb526e6130361b3d54203e84830c3cb947360b22e2e753d01c1","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}