{"meta":{"start_date":"2026-07-29","end_date":"2026-07-29","start_at":"2026-07-29T00:00:00Z","end_at":"2026-07-30T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T21:16:24Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":5,"failed_runs":4,"raw_occurrences":5,"matched_occurrences":5,"runs_affected":4},"rows":[{"environment":"stg","failure_pattern_id":"25717a9413da460dfb05353663d4d2350b470620920d197a41fa5e862fc8dbbc","failure_pattern":"failed to create service principal: create service principal: error status code received from the API","search_query":"failed to create service principal: create service principal: error status code received from the API","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-07-23..2026-07-29 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f047e0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f047e0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b724c0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b724c0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2082602235916718080","occurred_at":"2026-07-29T23:00:40Z","signature_id":"226ec60dda6e8dacd8b281d369d33c65671d533f56f7a7359de3bbd420e0861b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2082602235916718080","occurred_at":"2026-07-29T23:00:40Z","signature_id":"d9d5f099db5426c2db0a0a9c9736d118332550d2e849413beaf9e7d87762619c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"44e2d333ce62abafbaf16d046b61703140a463730eb39f1d9ff95ea9875c0757","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-23..2026-07-29 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2082285143308898304","occurred_at":"2026-07-29T02:00:39Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4e83fe1c3bb6daff1be6935129ccd1a1c67e021531f7a464980385c362ccc372","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9mjvf6c8tnwk\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9mjvf6c8tnwk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9mjvf6c8tnwk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9mjvf6c8tnwk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9mjvf6c8tnwk, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-23..2026-07-29 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":8,"prior_last_seen_at":"2026-07-26T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b7c240\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9mjvf6c8tnwk\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9mjvf6c8tnwk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9mjvf6c8tnwk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9mjvf6c8tnwk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9mjvf6c8tnwk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b7c240\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9mjvf6c8tnwk\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9mjvf6c8tnwk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9mjvf6c8tnwk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9mjvf6c8tnwk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9mjvf6c8tnwk, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2082549077467205632","occurred_at":"2026-07-29T19:29:26Z","signature_id":"c682b6a22dcdb927433beee37cadccbdd0d2b2db6b32b879d47be3e23e2b10c6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-23..2026-07-29 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010032a0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-dmn8zvx6vjjg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/255da654-b073-4883-9f2a-b049b05e0b86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/255da654-b073-4883-9f2a-b049b05e0b86\",\n      \"name\": \"255da654-b073-4883-9f2a-b049b05e0b86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T03:36:57.825472834Z\",\n      \"endTime\": \"2026-07-29T03:56:01.015852085Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010032a0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-dmn8zvx6vjjg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/255da654-b073-4883-9f2a-b049b05e0b86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/255da654-b073-4883-9f2a-b049b05e0b86\",\n      \"name\": \"255da654-b073-4883-9f2a-b049b05e0b86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T03:36:57.825472834Z\",\n      \"endTime\": \"2026-07-29T03:56:01.015852085Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2082302499082997760","occurred_at":"2026-07-29T03:09:37Z","signature_id":"57bb6252272e457ae8419a1eafbce66dcac93ff1eae74c35bb04b248a552b1b9","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}