{"meta":{"start_date":"2026-03-14","end_date":"2026-03-14","start_at":"2026-03-14T00:00:00Z","end_at":"2026-03-15T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T20:41:26Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":7,"failed_runs":4,"raw_occurrences":5,"matched_occurrences":5,"runs_affected":4},"rows":[{"environment":"int","failure_pattern_id":"8d06cad82b6f76b897e08d4e2537f6c6d4cc616ff64635c4d37783734841ca74","failure_pattern":"RevokeCredentialsAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-k2mdhb\" in resourcegroup=\"admin-credential-lifecycle-test-ddn92q\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-k2mdhb in resource group admin-credential-lifecycle-test-ddn92q, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":28.571428571428573,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-03-08..2026-03-14 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007e6a20\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-r44n87\" in resourcegroup=\"admin-credential-lifecycle-test-lgth9w\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-r44n87 in resource group admin-credential-lifecycle-test-lgth9w, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-r44n87\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-lgth9w\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-r44n87 in resource group admin-credential-lifecycle-test-lgth9w, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000916050\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-r44n87 in resource group admin-credential-lifecycle-test-lgth9w\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007e6a20\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-r44n87\" in resourcegroup=\"admin-credential-lifecycle-test-lgth9w\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-r44n87 in resource group admin-credential-lifecycle-test-lgth9w, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-r44n87\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-lgth9w\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-r44n87 in resource group admin-credential-lifecycle-test-lgth9w, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000916050\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-r44n87 in resource group admin-credential-lifecycle-test-lgth9w\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a485a0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-k2mdhb\" in resourcegroup=\"admin-credential-lifecycle-test-ddn92q\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-k2mdhb in resource group admin-credential-lifecycle-test-ddn92q, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-k2mdhb\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-ddn92q\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-k2mdhb in resource group admin-credential-lifecycle-test-ddn92q, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000ab8a00\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-k2mdhb in resource group admin-credential-lifecycle-test-ddn92q\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a485a0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-k2mdhb\" in resourcegroup=\"admin-credential-lifecycle-test-ddn92q\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-k2mdhb in resource group admin-credential-lifecycle-test-ddn92q, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-k2mdhb\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-ddn92q\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-k2mdhb in resource group admin-credential-lifecycle-test-ddn92q, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000ab8a00\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-k2mdhb in resource group admin-credential-lifecycle-test-ddn92q\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032842010720735232","occurred_at":"2026-03-14T15:31:18Z","signature_id":"971bff6eab10d7bbdfe66218da1fb2f68e2cb9ff1fcf9a098d59cd86f941c022","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032804286143926272","occurred_at":"2026-03-14T13:01:24Z","signature_id":"59f620953c130864f7871412b35f32f392d1be29509a3ce0429270c8d33d8aa7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"cedc818b7e720916031ed8d299fdf1177b8336a53376f99af83f86d0526a86bb","failure_pattern":"Admin credential 1 should fail after revocation, last error: \u003cnil\u003e","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-08..2026-03-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: Admin credential 1 should fail after revocation, last error: \u003cnil\u003e\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: Admin credential 1 should fail after revocation, last error: \u003cnil\u003e\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032871090371432448","occurred_at":"2026-03-14T17:26:51Z","signature_id":"d617bc01711b19fec7ae259e6b1da87ceb0b303c46e17ba4e53b8c013178e381","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"083a9cceffc299c1feee488fbf6249c7387be0a488330c96c1fef09ca1fc3da6","failure_pattern":"Timed out after 120.345s.","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-08..2026-03-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:168]: Timed out after 120.345s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:162 with:\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:168]: Timed out after 120.345s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:162 with:\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032610884349595648","occurred_at":"2026-03-14T00:12:53Z","signature_id":"d88c4a00af6acde1a56adecafa1580decbd06e9145355918ea48ded5553fde30","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-ccshfw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-08..2026-03-14 UTC","prior_windows_present":2,"prior_window_starts":["2026-02-23","2026-03-02"],"prior_runs_affected":16,"prior_last_seen_at":"2026-03-08T23:23:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009aed80\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-ccshfw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-ccshfw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f45af0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0009aed50\u003e{\n                msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-ccshfw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f45af0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009aed80\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-ccshfw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-ccshfw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f45af0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0009aed50\u003e{\n                msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-ccshfw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f45af0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032610884349595648","occurred_at":"2026-03-14T00:12:53Z","signature_id":"37269be883771bcdfa1b48c6a53e74a57dd276f0cb9c6f8b96f16a957b572578","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}