{"meta":{"start_date":"2026-07-12","end_date":"2026-07-12","start_at":"2026-07-12T00:00:00Z","end_at":"2026-07-13T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T20:20:47Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":3,"failed_runs":2,"raw_occurrences":3,"matched_occurrences":3,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"4e83fe1c3bb6daff1be6935129ccd1a1c67e021531f7a464980385c362ccc372","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-4-22-p27992\", caused by: failed waiting for hcpCluster=\"cluster-candidate-4-22-bf67pt\" in resourcegroup=\"rg-candidate-4-22-p27992\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-bf67pt in resource group rg-candidate-4-22-p27992, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-4-22-bf67pt\" in resourcegroup=\"rg-candidate-4-22-p27992\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-bf67pt in resource group rg-candidate-4-22-p27992, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[0,0,1,0,5,0,2],"trend_range":"2026-07-06..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00108a450\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-4-22-p27992\", caused by: failed waiting for hcpCluster=\"cluster-candidate-4-22-bf67pt\" in resourcegroup=\"rg-candidate-4-22-p27992\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-bf67pt in resource group rg-candidate-4-22-p27992, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-4-22-bf67pt\" in resourcegroup=\"rg-candidate-4-22-p27992\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-bf67pt in resource group rg-candidate-4-22-p27992, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00108a450\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-4-22-p27992\", caused by: failed waiting for hcpCluster=\"cluster-candidate-4-22-bf67pt\" in resourcegroup=\"rg-candidate-4-22-p27992\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-bf67pt in resource group rg-candidate-4-22-p27992, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-4-22-bf67pt\" in resourcegroup=\"rg-candidate-4-22-p27992\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-bf67pt in resource group rg-candidate-4-22-p27992, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b34240\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-labels-taints-n2l8g2\", caused by: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-n2l8g2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-n2l8g2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-n2l8g2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-n2l8g2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b34240\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-labels-taints-n2l8g2\", caused by: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-n2l8g2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-n2l8g2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-n2l8g2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-n2l8g2, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2076105588294553600","occurred_at":"2026-07-12T00:45:19Z","signature_id":"89d831ccf72982f39cc5f6ca2e54099b1d26c1bf2196c5591db6867b190d7b70","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2076105588294553600","occurred_at":"2026-07-12T00:45:19Z","signature_id":"a18f0e5dd08f60d5164e679f2c87187f311909570648924226560465b75246ea","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b64b4ef0c88d35afab6c6cd8765ff6a9da8a5a8ffc9a639da00c5e1f9e608e45","failure_pattern":"ERROR CODE: NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/e2e-oidc-wi-test' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-06..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:296]: failed to create federated identity credential for service account system:serviceaccount:e2e-oidc-wi:wi-test-sa\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0024efa70\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-m6pkth/providers/Microsoft.ManagedIdentity/userAssignedIdentities/e2e-oidc-wi-test/federatedIdentityCredentials/e2e-wi-fic\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NotFound\",\n        \"message\": \"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/oidc-wi-m6pkth/providers/Microsoft.ManagedIdentity/userAssignedIdentities/e2e-oidc-wi-test' was not found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:296]: failed to create federated identity credential for service account system:serviceaccount:e2e-oidc-wi:wi-test-sa\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0024efa70\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-m6pkth/providers/Microsoft.ManagedIdentity/userAssignedIdentities/e2e-oidc-wi-test/federatedIdentityCredentials/e2e-wi-fic\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NotFound\",\n        \"message\": \"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/oidc-wi-m6pkth/providers/Microsoft.ManagedIdentity/userAssignedIdentities/e2e-oidc-wi-test' was not found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2076124508074283008","occurred_at":"2026-07-12T02:00:29Z","signature_id":"697ed4f1fa31f906da6e81d1dba09423da734b4de942cfb34f304a40f53ca43b","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}