{"meta":{"start_date":"2026-04-01","end_date":"2026-04-01","start_at":"2026-04-01T00:00:00Z","end_at":"2026-04-02T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T16:11:05Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":10,"matched_occurrences":10,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-7bwpw7\" in resourcegroup=\"admin-credential-lifecycle-test-ms8t98\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-7bwpw7 in resource group admin-credential-lifecycle-test-ms8t98, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":9,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":9,"after_last_push_count":11,"trend_counts":[0,0,0,0,0,0,9],"trend_range":"2026-03-26..2026-04-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":2,"prior_last_seen_at":"2026-03-23T02:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00115e6f0\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-sk5knd\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sk5knd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-sk5knd\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sk5knd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000238640\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sk5knd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00115e6f0\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-sk5knd\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sk5knd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-sk5knd\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sk5knd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000238640\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sk5knd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f8a570\u003e: \n    failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5f42rl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5f42rl, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-5f42rl\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5f42rl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009a6400\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5f42rl\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f8a570\u003e: \n    failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5f42rl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5f42rl, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-5f42rl\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5f42rl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009a6400\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5f42rl\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e26bd0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-7bwpw7\" in resourcegroup=\"admin-credential-lifecycle-test-ms8t98\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-7bwpw7 in resource group admin-credential-lifecycle-test-ms8t98, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-7bwpw7\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-ms8t98\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-7bwpw7 in resource group admin-credential-lifecycle-test-ms8t98, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000aea030\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-7bwpw7 in resource group admin-credential-lifecycle-test-ms8t98\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e26bd0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-7bwpw7\" in resourcegroup=\"admin-credential-lifecycle-test-ms8t98\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-7bwpw7 in resource group admin-credential-lifecycle-test-ms8t98, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-7bwpw7\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-ms8t98\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-7bwpw7 in resource group admin-credential-lifecycle-test-ms8t98, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000aea030\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-7bwpw7 in resource group admin-credential-lifecycle-test-ms8t98\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"99c415edd6b95eec8761b3671d6423a3a146ba7c00178112a04e67159626c816","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"9b101c4a736d523bb15934eea086cc5ca2341cec5f5bfab68fe667ee172a4857","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"b3da81362b9698cffdac12e8ad5fd90a828a899d06d074781eeb6cefeafa8ec4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"b8e8fab0cce09ad90ef2251627883aa3bf86e27e2c20dbe95e2cea30446f507a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"bbad72fda5e343074c8c3e32f9cba154886471d76368cdd557ef9784681db104","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"c3b9140d3fdfc6f04844c1305bbffd8c10e94819b3a4ebe810b753e959596bf0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"d18e01fea11974ed0eba492770e2f8a7d174f753a3b28796e1137234faeb8272","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"f88f46b79d01481827ca3fd0763047f1a5e1f65a35470b8d19a7ff3d98a3330b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"fcf6f271f147ca0d27070d73d4c1265abf1b5b47346a91a590dc07c731bfb2f6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"874f7f16a0de76992185fc0a72b94dc6fc102bf27a7c4b1aa28301f1025b5686","failure_pattern":"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8mw26m","search_query":"occurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-26..2026-04-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246]: Unexpected error:\n    \u003c*errors.joinError | 0xc00161a870\u003e: \n    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8mw26m\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc001950750\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8mw26m\",\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246]: Unexpected error:\n    \u003c*errors.joinError | 0xc00161a870\u003e: \n    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8mw26m\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc001950750\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8mw26m\",\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039475678092464128","occurred_at":"2026-04-01T22:51:07Z","signature_id":"efe40ef3c88aaa17efe2662e93d8b30c200b0134f9de3ca8fce697b912e390a4","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}