{"meta":{"start_date":"2026-06-22","end_date":"2026-06-28","start_at":"2026-06-22T00:00:00Z","end_at":"2026-06-29T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T21:17:25Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":29,"failed_runs":22,"raw_occurrences":61,"matched_occurrences":61,"runs_affected":22},"rows":[{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-kxcsr7, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-kxcsr7 in resource group rg-candidate-4-20-79p2x6, error: failed waiting for cluster=\"cluster-candidate-4-20-kxcsr7\" in resourcegroup=\"rg-candidate-4-20-79p2x6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-kxcsr7 in resource group rg-candidate-4-20-79p2x6, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":35,"runs_affected":15,"failed_runs":15,"run_impact_percent":51.724137931034484,"anchor_occurrences":35,"after_last_push_count":35,"trend_counts":[0,5,12,6,3,0,7,2],"trend_range":"2026-06-21..2026-06-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-25","2026-06-01","2026-06-08","2026-06-15"],"prior_runs_affected":47,"prior_last_seen_at":"2026-06-20T23:00:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-7bcwz9/cluster-candidate-4-22-wmxsmt should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a1c1b0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-wmxsmt, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-wmxsmt in resource group rg-candidate-4-22-7bcwz9, error: failed waiting for cluster=\"cluster-candidate-4-22-wmxsmt\" in resourcegroup=\"rg-candidate-4-22-7bcwz9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-wmxsmt in resource group rg-candidate-4-22-7bcwz9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-7bcwz9/cluster-candidate-4-22-wmxsmt should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a1c1b0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-wmxsmt, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-wmxsmt in resource group rg-candidate-4-22-7bcwz9, error: failed waiting for cluster=\"cluster-candidate-4-22-wmxsmt\" in resourcegroup=\"rg-candidate-4-22-7bcwz9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-wmxsmt in resource group rg-candidate-4-22-7bcwz9, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-wg5rzb/cluster-candidate-5-0-4q5ggx should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014f0720\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-4q5ggx, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-4q5ggx in resource group rg-candidate-5-0-wg5rzb, error: failed waiting for cluster=\"cluster-candidate-5-0-4q5ggx\" in resourcegroup=\"rg-candidate-5-0-wg5rzb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-4q5ggx in resource group rg-candidate-5-0-wg5rzb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-wg5rzb/cluster-candidate-5-0-4q5ggx should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014f0720\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-4q5ggx, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-4q5ggx in resource group rg-candidate-5-0-wg5rzb, error: failed waiting for cluster=\"cluster-candidate-5-0-4q5ggx\" in resourcegroup=\"rg-candidate-5-0-wg5rzb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-4q5ggx in resource group rg-candidate-5-0-wg5rzb, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-22-858hzm/cluster-nightly-4-22-xcdlzz should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001162390\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-xcdlzz, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-22-xcdlzz in resource group rg-nightly-4-22-858hzm, error: failed waiting for cluster=\"cluster-nightly-4-22-xcdlzz\" in resourcegroup=\"rg-nightly-4-22-858hzm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-22-xcdlzz in resource group rg-nightly-4-22-858hzm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-22-858hzm/cluster-nightly-4-22-xcdlzz should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001162390\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-xcdlzz, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-22-xcdlzz in resource group rg-nightly-4-22-858hzm, error: failed waiting for cluster=\"cluster-nightly-4-22-xcdlzz\" in resourcegroup=\"rg-nightly-4-22-858hzm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-22-xcdlzz in resource group rg-nightly-4-22-858hzm, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071331624070418432","occurred_at":"2026-06-28T20:35:17Z","signature_id":"3f7243f8e285c3f9ecbc6d7fad6b00889350dcd67ac766d3ebda3bbe9ccb61f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071331624070418432","occurred_at":"2026-06-28T20:35:17Z","signature_id":"65b7a10b456bbd873078f4fd4c9d0ab632c9dea1e8097b09eb788630e5be2fce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071005750192246784","occurred_at":"2026-06-27T23:00:22Z","signature_id":"3916e2bc6430355b0c60c2543017aeed47ae5f657c113ef0bde07bfa49f0bad6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071005750192246784","occurred_at":"2026-06-27T23:00:22Z","signature_id":"55587457e02b6f1c3be2cbd1cc374b1b5eca347ffabf150ea59dba47fc309b93","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071005750192246784","occurred_at":"2026-06-27T23:00:22Z","signature_id":"7f40e2a6fe7e5020055cc8364d432b90f01b48977401edc2feebea68356477f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071005750192246784","occurred_at":"2026-06-27T23:00:22Z","signature_id":"af0353fe4595ded5ba48e069f76340f1258a4b004edf0d1055b73b62725a1bd1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071005750192246784","occurred_at":"2026-06-27T23:00:22Z","signature_id":"dfdd71101eccd6d166ffc445fa7d2ee4f1aa1101c5afbcefa9665dc962573cd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071005750192246784","occurred_at":"2026-06-27T23:00:22Z","signature_id":"e874239b71dad1df108965fe415f13fae872bb216ea04c184e029cd0ce4cfb94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071005750192246784","occurred_at":"2026-06-27T23:00:22Z","signature_id":"eb3937b0d5bf24ff67b1e0b2878ec94f93a151517d7dcc25c7d1d5f7c7061457","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2070280913878519808","occurred_at":"2026-06-25T23:00:08Z","signature_id":"515cfde894652e5d2bf00f29239afcc5194cafeffecf19a84ce0782115dd87a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070093862793121792","occurred_at":"2026-06-25T10:36:51Z","signature_id":"210f3648e7a374ef3b314d400049eee0704dddd28a091adf15a63b78a73994fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070093862793121792","occurred_at":"2026-06-25T10:36:51Z","signature_id":"74ec772d4b959eff888e30dd08c73ea14b8ae600fd7df1398191910740a3c94f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069918523580223488","occurred_at":"2026-06-24T23:00:07Z","signature_id":"66d22edd2eabbede740e0ae74801b7b4028a80f398e9efe309f025faf5b1de3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069918523580223488","occurred_at":"2026-06-24T23:00:07Z","signature_id":"7f23c32205e009f2513fa4394baeee341c6e3a46682094fbd58fd487624df2f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069918523580223488","occurred_at":"2026-06-24T23:00:07Z","signature_id":"91316ac4fdf62cb0ca5a89e1dfb729ec3a50f080fcb9822b88694c50fb95d0f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069918523580223488","occurred_at":"2026-06-24T23:00:07Z","signature_id":"cecf8431fe2070bced0374024c3b3a9aa88323404eb395f50595abdf48f30e17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069918523580223488","occurred_at":"2026-06-24T23:00:07Z","signature_id":"d68d6d3f013dea73bbf3ecd2d5a5578d66d1c666f00cae1d7fc8bf57cf474cff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069650559614324736","occurred_at":"2026-06-24T05:15:20Z","signature_id":"a2e982b263705c31d409871a3fd382bbc022f76552e3462b41ab9c94155e9e7b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"08dde14e0755e6ea5b06fb014131678e7fc05e185482f637b523df0497a0d17d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"2582df8cbd124d0d4a2afb9fba60b204fc53a7e20cd4a767a81f1dc6d5a109ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"46d389d69ba42fef1cc8ad1bb6252c4b6f97d5695dee2b1efd6a91ebb364a6c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"9d2e37a41ac31e236364c0b5b397cde40ae39ceb2e35d9503711f98fb79c863b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"b1be0ad549d21afa914b5d035cac2ffad0abaea30acb1bd837d520abc999f275","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"c498bbc4ca442add49dd1958a514c1156eebc818c8194225bed803c7d97580ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"ca448ffc21aca0ff90acd1329ed31e2286331f478238276c000031e6a14ac104","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069417000056131584","occurred_at":"2026-06-23T13:46:50Z","signature_id":"2a2d884f4501c77edc8bcbde4a690b46b876202da711d3c46da82c5214c89ec7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069264571155091456","occurred_at":"2026-06-23T03:41:33Z","signature_id":"2008f8bdf7e6770f61faf0871a346f51f3345fcfb4ed32d1cdfaa7d13d267976","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069264571155091456","occurred_at":"2026-06-23T03:41:33Z","signature_id":"3946fab07c0358b15d5ea4fab6fc2928896b66fabf448208ac28abdef2f554c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2069239037184446464","occurred_at":"2026-06-23T02:00:05Z","signature_id":"6dc6abb6d49682cf350aed62faed9312f2b8be133716fe5adebe992063546d53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069230875538624512","occurred_at":"2026-06-23T01:27:39Z","signature_id":"ca66aa298134cfa257dfb7f99c1e390a42943ce646f440e0820b5ca0b7d2aad1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069193738747383808","occurred_at":"2026-06-22T23:00:05Z","signature_id":"b499dfd9e3a9bdd7f99096eb91aa8936a5522c33190c6da69f6dd19013892363","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069071579051986944","occurred_at":"2026-06-22T14:54:35Z","signature_id":"53021f426ac4e51d3cb6f45bee956b84c3f109e0d013a1d7786d79029ca52a86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069002705430384640","occurred_at":"2026-06-22T10:20:59Z","signature_id":"7ef0493f254c06f948fbb27c1e5f409fe6455019f3ee1b4e053c6965c5af13ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069002705430384640","occurred_at":"2026-06-22T10:20:59Z","signature_id":"cb0cbe311134b6ebcefbfb792a20a8cefb1dc9ab6e7dbee066b9898fc5752e2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2068971202583465984","occurred_at":"2026-06-22T08:15:48Z","signature_id":"4e453a2e9360183edfba139347d397418ae5d38eff0fca154ae702111c3f49fa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nfhflv\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":13,"runs_affected":9,"failed_runs":9,"run_impact_percent":31.03448275862069,"anchor_occurrences":13,"after_last_push_count":13,"trend_counts":[0,3,0,0,2,5,1,2],"trend_range":"2026-06-21..2026-06-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-01","2026-06-08","2026-06-15"],"prior_runs_affected":13,"prior_last_seen_at":"2026-06-19T23:00:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00130e300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nfhflv\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00130e300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nfhflv\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013be438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-nr68nw\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013be438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-nr68nw\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0020020f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-v7fdgp\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0020020f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-v7fdgp\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071368141530730496","occurred_at":"2026-06-28T23:00:23Z","signature_id":"b47c33be2132af3d48dfa29add64496488d3c05a3948bf68e839d294e89f1e9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071331624070418432","occurred_at":"2026-06-28T20:35:17Z","signature_id":"7ff6a8cd5d1a64c38ebd4d68a4adbd5cffeffe8d922069aab0ab21e5ef680c2c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2070688658855301120","occurred_at":"2026-06-27T02:00:22Z","signature_id":"c08749b4bad949ea96508ff1b671f09a2d4c4eb8c754e4d2fe916340f4ade6ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070562411533832192","occurred_at":"2026-06-26T17:38:42Z","signature_id":"6c3d2c759388db4bd85af18b6309fc528d75f435de0f4458776417b850e485fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070509055310303232","occurred_at":"2026-06-26T14:06:41Z","signature_id":"06d47ff977f52897bc5e007d579a0916d2af935e44254cd67ab80f1cac6ce7ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070509055310303232","occurred_at":"2026-06-26T14:06:41Z","signature_id":"232e2ee071a4250eefa690c19891fcdc9efe33a1598efd2b1b43a97f558a07e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070509055310303232","occurred_at":"2026-06-26T14:06:41Z","signature_id":"3b8549e54ccbd048c39382013b9cd01e12755b8968e4682a82dd827b5f51bb97","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070509055310303232","occurred_at":"2026-06-26T14:06:41Z","signature_id":"cda641f267925626014efe3280d8d21b80a86f8d65ec49afd1f58d35c466451b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070093862793121792","occurred_at":"2026-06-25T10:36:51Z","signature_id":"d677947a3befdfce6af5dffcbae4e0388155689f9c7b64fb547f4fba0c15273d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069966174292021248","occurred_at":"2026-06-25T02:09:28Z","signature_id":"1e784ab3fdf51282a18ce376758233e043051061c77fbde9af79b4fcd98b8e7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069193738747383808","occurred_at":"2026-06-22T23:00:05Z","signature_id":"cf0298768bf3d36c59220c7a0cfb551998ec62877185c8fc3712300cbfb4c581","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069193738747383808","occurred_at":"2026-06-22T23:00:05Z","signature_id":"f3fabf19c99a298b418328ac0114b6d63c61ee4e7af72157649bb72b8a5321b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069071579051986944","occurred_at":"2026-06-22T14:54:35Z","signature_id":"c8abacb525c80949e625916c338d8af12373d1f272cda34977ee1cabc43ca8e6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-bxgl8t, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bxgl8t\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":8,"runs_affected":7,"failed_runs":7,"run_impact_percent":24.137931034482758,"anchor_occurrences":8,"after_last_push_count":20,"trend_counts":[0,1,1,0,0,2,2,2],"trend_range":"2026-06-21..2026-06-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-25","2026-06-08","2026-06-15"],"prior_runs_affected":20,"prior_last_seen_at":"2026-06-19T19:37:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-qt62hg\" for cluster \"cluster-nightly-4-22-qt62hg\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001963170\u003e: \n    failed to create NodePool np-qt62hg, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-qt62hg in resource group rg-nightly-4-22-89k865, error: failed waiting for nodepool=\"np-qt62hg\" for cluster \"cluster-nightly-4-22-qt62hg\" in resourcegroup=\"rg-nightly-4-22-89k865\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-qt62hg\" for cluster \"cluster-nightly-4-22-qt62hg\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001963170\u003e: \n    failed to create NodePool np-qt62hg, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-qt62hg in resource group rg-nightly-4-22-89k865, error: failed waiting for nodepool=\"np-qt62hg\" for cluster \"cluster-nightly-4-22-qt62hg\" in resourcegroup=\"rg-nightly-4-22-89k865\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:214]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a33ad0\u003e: \n    failed to create NodePool autoscale-noaz, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-f2qq9q, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-f2qq9q\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:214]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a33ad0\u003e: \n    failed to create NodePool autoscale-noaz, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-f2qq9q, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-f2qq9q\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:168]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0026879e0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-negative-tests-7584j7, error: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-7584j7\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:168]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0026879e0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-negative-tests-7584j7, error: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-7584j7\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071368141530730496","occurred_at":"2026-06-28T23:00:23Z","signature_id":"441b6d97911f886e0224e335cee97dd32f5ac764d1ffb467f983f3add45058f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071331624070418432","occurred_at":"2026-06-28T20:35:17Z","signature_id":"827ec00a4ccbc057a79d910fdc1accdc3d7590478989dd3b9f81d7a8975b5b82","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2070688658855301120","occurred_at":"2026-06-27T02:00:22Z","signature_id":"3e6cfa8938304633cb8f796463bb341c509ad28b67cd8a8cf3d95caa3d6c07dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2070688658855301120","occurred_at":"2026-06-27T02:00:22Z","signature_id":"52f393ae31f6d9df41d2ee09a0c23c5d372fb97e53cdf571a9a1fe008d9c8667","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2070643359336108032","occurred_at":"2026-06-26T23:00:22Z","signature_id":"68fc3e48c83cc3669caa353ef444c437b12c5d584123130a4b771cbca76dd6fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070509055310303232","occurred_at":"2026-06-26T14:06:41Z","signature_id":"16854ae51e6191c92e5ef37ad7f1a1bd92bffcdfd7d2a9b3f1aab3cf686d3ee6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069417000056131584","occurred_at":"2026-06-23T13:46:50Z","signature_id":"9d41f5140c8f9c893570641b224ac88b48ff81c1c5266bea4d485a0498d7d8ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069193738747383808","occurred_at":"2026-06-22T23:00:05Z","signature_id":"efbd50b9b067f5293472532fae2e3ee43590f38d2028e4a53e161b1bfa71b808","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.344827586206897,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,1,1,0,1,0,0],"trend_range":"2026-06-21..2026-06-28 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-c6rlpk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f740a0\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-gxxlm.apps.aro.g0n7d1z2u8n7b0t.0wrk.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-c6rlpk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f740a0\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-gxxlm.apps.aro.g0n7d1z2u8n7b0t.0wrk.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00151a240\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-2prdk.apps.aro.cilium-cl.zqsw.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00151a240\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-2prdk.apps.aro.cilium-cl.zqsw.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000881980\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-qfrzh.apps.aro.cilium-cl.9zjy.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000881980\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-qfrzh.apps.aro.cilium-cl.9zjy.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070562411533832192","occurred_at":"2026-06-26T17:38:42Z","signature_id":"4ca95a24ded3a119fb9dbf11c955f9e65212aabc4ba5547a61850eddf39d6d52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069918523580223488","occurred_at":"2026-06-24T23:00:07Z","signature_id":"02215f8b0cf3a0ffd266c0db13e29161d4e56c9c72afccc01ecb3030a30ef127","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069230875538624512","occurred_at":"2026-06-23T01:27:39Z","signature_id":"c7d4b42baef0d672054b5dbb2d77c68db076879abc452803eeb38b332bb67ec5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"078c8370f997bf97070c98ca66b6e31b17e4874c4cf5b608a229a824de3b303d","failure_pattern":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-06-21..2026-06-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:269]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d45830\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:269]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d45830\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069002705430384640","occurred_at":"2026-06-22T10:20:59Z","signature_id":"ea7f7ea2b3e749de6059661f9c0bd4c2ee23fa07bab3900cf551780158d30fd8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b67cae5d7b1734c61859c2df1079b54448a4d5968fa6f6cea032c780834dc503","failure_pattern":"expected \u003ccount\u003e ready (and schedulable) nodes, found \u003ccount\u003e","search_query":"expected 3 ready (and schedulable) nodes in node pool \"npupgrade-4-20\", found 4","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-06-21..2026-06-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231]: failed to verify 3 ready and schedulable nodes for nodepool npupgrade-4-20 after upgrade\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000f85a50\u003e: \n    expected 3 ready (and schedulable) nodes in node pool \"npupgrade-4-20\", found 4\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231]: failed to verify 3 ready and schedulable nodes for nodepool npupgrade-4-20 after upgrade\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000f85a50\u003e: \n    expected 3 ready (and schedulable) nodes in node pool \"npupgrade-4-20\", found 4\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2068876744932200448","occurred_at":"2026-06-22T02:00:28Z","signature_id":"dc74b26189a451aa3afa2fb79f8978fccc65d244edcdd6b89d697215b555f044","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}