{"meta":{"start_date":"2026-04-20","end_date":"2026-04-26","start_at":"2026-04-20T00:00:00Z","end_at":"2026-04-27T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T13:39:38Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":29,"failed_runs":13,"raw_occurrences":21,"matched_occurrences":20,"runs_affected":12},"rows":[{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-64zz5k, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-64zz5k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-64zz5k, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":13.793103448275861,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,1,0,3,1,0,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":4,"prior_window_starts":["2026-03-23","2026-03-30","2026-04-06","2026-04-13"],"prior_runs_affected":22,"prior_last_seen_at":"2026-04-19T20:51:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012e0690\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-f9lgb5, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-f9lgb5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-f9lgb5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012e0690\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-f9lgb5, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-f9lgb5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-f9lgb5, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00110ea50\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5n8ccc, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5n8ccc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5n8ccc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00110ea50\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5n8ccc, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5n8ccc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5n8ccc, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001880b70\u003e: \n    failed to create HCP cluster cilium-cl, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-bljg24, error: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-bljg24\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-bljg24, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001880b70\u003e: \n    failed to create HCP cluster cilium-cl, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-bljg24, error: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-bljg24\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-bljg24, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2047432489114275840","occurred_at":"2026-04-23T21:48:39Z","signature_id":"a745674107cf2356831faa99da512bc05606e802446b082758661703b09d8e99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046962898504257536","occurred_at":"2026-04-22T14:42:40Z","signature_id":"ccd1adb6d2efec6a7e37e7a5a67c25905b5ea7f9b00a002db2df895e092d738b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046885871331840000","occurred_at":"2026-04-22T09:36:35Z","signature_id":"271cfae3f86d85a5adc5299fa582d276e24dfb97ea7940d98539545058e6a3f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046885871331840000","occurred_at":"2026-04-22T09:36:35Z","signature_id":"c5b144647101777ff757a6fd42947d83c706b18038b52993de2d33242ea25f64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046366938937954304","occurred_at":"2026-04-20T23:14:32Z","signature_id":"e7447258bc07df0854c9f9794e99968221271e6698de5950098f8c78cdb1abad","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0229ee4241b7de7313eea7ac47ba9b3898c5249b1780e4f26a10331bd5c30790","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.i0i5b4m4e4n6i2h.usea.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.i0i5b4m4e4n6i2h.n82v.uksouth.aroapp-hcp.azure-test.net","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.344827586206897,"anchor_occurrences":4,"after_last_push_count":5,"trend_counts":[0,1,1,2,0,0,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-06"],"prior_runs_affected":4,"prior_last_seen_at":"2026-04-11T04:17:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e8c0c0\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.i2o7e9m2u0f5e5k.kyxh.uksouth.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.i2o7e9m2u0f5e5k.zser.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.i2o7e9m2u0f5e5k.kyxh.uksouth.aroapp-hcp.azure-test.net\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.i2o7e9m2u0f5e5k.kyxh.uksouth.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.i2o7e9m2u0f5e5k.zser.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.i2o7e9m2u0f5e5k.kyxh.uksouth.aroapp-hcp.azure-test.net\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e8c0c0\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.i2o7e9m2u0f5e5k.kyxh.uksouth.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.i2o7e9m2u0f5e5k.zser.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.i2o7e9m2u0f5e5k.kyxh.uksouth.aroapp-hcp.azure-test.net\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.i2o7e9m2u0f5e5k.kyxh.uksouth.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.i2o7e9m2u0f5e5k.zser.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.i2o7e9m2u0f5e5k.kyxh.uksouth.aroapp-hcp.azure-test.net\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f06ff0\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.m4s3u7n2e5l1m2p.nay8.uksouth.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.m4s3u7n2e5l1m2p.kj53.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.m4s3u7n2e5l1m2p.nay8.uksouth.aroapp-hcp.azure-test.net\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.m4s3u7n2e5l1m2p.nay8.uksouth.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.m4s3u7n2e5l1m2p.kj53.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.m4s3u7n2e5l1m2p.nay8.uksouth.aroapp-hcp.azure-test.net\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f06ff0\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.m4s3u7n2e5l1m2p.nay8.uksouth.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.m4s3u7n2e5l1m2p.kj53.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.m4s3u7n2e5l1m2p.nay8.uksouth.aroapp-hcp.azure-test.net\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.m4s3u7n2e5l1m2p.nay8.uksouth.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.m4s3u7n2e5l1m2p.kj53.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.m4s3u7n2e5l1m2p.nay8.uksouth.aroapp-hcp.azure-test.net\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:141]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010a2f78\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.x2h2g9i6z7r1v3a.cwij.uksouth.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.x2h2g9i6z7r1v3a.r03g.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.x2h2g9i6z7r1v3a.cwij.uksouth.aroapp-hcp.azure-test.net\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.x2h2g9i6z7r1v3a.cwij.uksouth.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.x2h2g9i6z7r1v3a.r03g.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.x2h2g9i6z7r1v3a.cwij.uksouth.aroapp-hcp.azure-test.net\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:141]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010a2f78\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.x2h2g9i6z7r1v3a.cwij.uksouth.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.x2h2g9i6z7r1v3a.r03g.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.x2h2g9i6z7r1v3a.cwij.uksouth.aroapp-hcp.azure-test.net\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.x2h2g9i6z7r1v3a.cwij.uksouth.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.x2h2g9i6z7r1v3a.r03g.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.x2h2g9i6z7r1v3a.cwij.uksouth.aroapp-hcp.azure-test.net\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046841243291357184","occurred_at":"2026-04-22T06:39:15Z","signature_id":"6035092a2b04a4c82bf69583ecf0fbc5f52c484bd574df16b77c0ed7fa0c29a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046841243291357184","occurred_at":"2026-04-22T06:39:15Z","signature_id":"78fe18ef72526987d33303f033090e80a75b2b2bce64c1641b90f5924fad5bb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046563811896332288","occurred_at":"2026-04-21T12:16:50Z","signature_id":"3270078bc44ff5fc6509f068e30c6ed0b1bc2b486e691ff42b21f7633d8e48e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046366938937954304","occurred_at":"2026-04-20T23:14:32Z","signature_id":"f6302d55e4f998808fc14e78a431a5d9b8960ebc0a53e095988026b534ad91b0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"ed91849277e23f84f2cfe3e0bb2a2379781cf9007095371a6220d51d0605b4b7","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.896551724137931,"anchor_occurrences":3,"after_last_push_count":9,"trend_counts":[0,2,0,1,0,0,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-23","2026-04-06"],"prior_runs_affected":18,"prior_last_seen_at":"2026-04-10T04:37:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:171]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:171]: Interrupted by User","fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046927318231814144","occurred_at":"2026-04-22T12:21:17Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046283743831789568","occurred_at":"2026-04-20T17:42:36Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046283743831789568","occurred_at":"2026-04-20T17:42:36Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"88fb289639ac585d41e102922a6ea4f479e947e9962ca681abe481db2908b42c","failure_pattern":"context deadline exceeded","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.896551724137931,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,1,1,0,0,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046962898504257536","occurred_at":"2026-04-22T14:42:40Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046514545698017280","occurred_at":"2026-04-21T09:00:58Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"2d1906ec85d934b5bfe9201fdd489d4450037751378f0d66a8abd6c1b1f5a4bd","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":11,"prior_last_seen_at":"2026-03-27T08:52:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046885871331840000","occurred_at":"2026-04-22T09:36:35Z","signature_id":"8a441c3cc9a1724fd565ecc68f397cdfdb078c056f41b0ab5661bd4e719eb328","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"11627fd97b445c2f08e2be8ae642dc8d395499056ca9309dd64efc606ab993b2","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Cluster '\u003cid\u003e' is in state 'pending_update', can't update; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":2,"prior_window_starts":["2026-04-06","2026-04-13"],"prior_runs_affected":5,"prior_last_seen_at":"2026-04-18T01:00:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:231]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc0008ea4e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xs72kc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Cluster '2ps4n3477lh60n0ap813n047u1c1s32b' is in state 'pending_update', can't update\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:231]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc0008ea4e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xs72kc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Cluster '2ps4n3477lh60n0ap813n047u1c1s32b' is in state 'pending_update', can't update\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2047432489114275840","occurred_at":"2026-04-23T21:48:39Z","signature_id":"9e7c6f81b49d129b3f718f74f3ddf0a21cbf5a18430fa71da2c1326c03585eff","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"13836948ef96e4246c19567e32299b4bbf071749bdf2ad75c3dfd29f2a06679e","failure_pattern":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","search_query":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2047167290578833408","occurred_at":"2026-04-23T04:14:51Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"cf6d2d52d407628ab74bef9bd0acb08a16b0b9b7497165431ceaef2b75b3b183","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:109]: Expected success, but got an error:\n    \u003c*errors.joinError | 0xc000a23ef0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.cluster-gpu-np.iw7u.uksouth.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.cluster-gpu-np.w88z.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.cluster-gpu-np.iw7u.uksouth.aroapp-hcp.azure-test.net\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.cluster-gpu-np.iw7u.uksouth.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.cluster-gpu-np.w88z.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.cluster-gpu-np.iw7u.uksouth.aroapp-hcp.azure-test.net\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:109]: Expected success, but got an error:\n    \u003c*errors.joinError | 0xc000a23ef0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.cluster-gpu-np.iw7u.uksouth.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.cluster-gpu-np.w88z.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.cluster-gpu-np.iw7u.uksouth.aroapp-hcp.azure-test.net\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.cluster-gpu-np.iw7u.uksouth.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.cluster-gpu-np.w88z.uksouth.aroapp-hcp.azure-test.net, reserved.aroapp-hcp.azure-test.net, not api.cluster-gpu-np.iw7u.uksouth.aroapp-hcp.azure-test.net\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046283743831789568","occurred_at":"2026-04-20T17:42:36Z","signature_id":"a846c795ddc0689cc486b56d36c2cfcf86a344961eaba6af46f2d7829c1f9628","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-13"],"prior_runs_affected":2,"prior_last_seen_at":"2026-04-18T04:18:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000aba880\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-hqcpw.apps.aro.cilium-cluster.u8xb.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000aba880\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-hqcpw.apps.aro.cilium-cluster.u8xb.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2046399283279695872","occurred_at":"2026-04-21T01:23:04Z","signature_id":"9d7da680da5f0454642831eadb8d70ae16a9d8579bb5ef407ff5ec554b24fa86","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"77fb076d32302f92d929786f345bdad3139a0df750404c0633de671aa620060b","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-30"],"prior_runs_affected":2,"prior_last_seen_at":"2026-04-02T15:40:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:153]: Timed out after 240.003s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:153]: Timed out after 240.003s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2047004090818367488","occurred_at":"2026-04-22T17:26:21Z","signature_id":"0b527fdee65049cfb375368fd49dd24cf8d231282b78ffc0050ed1cd95e54b83","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}