{"meta":{"start_date":"2026-10-26","end_date":"2026-11-01","start_at":"2026-10-26T00:00:00Z","end_at":"2026-11-02T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T02:11:52Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":17,"failed_runs":10,"raw_occurrences":27,"matched_occurrences":24,"runs_affected":7},"rows":[{"environment":"stg","failure_pattern_id":"a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":10,"after_last_push_count":35,"trend_counts":[0,0,10,0,0,0,0,0],"trend_range":"2026-10-25..2026-11-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":16,"prior_last_seen_at":"2026-10-05T23:00:54Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-jxwc9m\" for cluster \"cluster-candidate-5-1-jxwc9m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2089be556480\u003e: \n    failed to create NodePool np-jxwc9m: failed waiting for nodepool=\"np-jxwc9m\" for cluster \"cluster-candidate-5-1-jxwc9m\" in resourcegroup=\"rg-candidate-5-1-t5bnkh9ptx8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\",\n      \"name\": \"aa13a852-544e-4ae1-9ee9-db477af0802a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:36:20.928028726Z\",\n      \"endTime\": \"2026-10-27T08:55:28.505111283Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-nmkn7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-rxvtq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-nmkn7: Creating: virtualmachine creating or updating\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-rxvtq: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\n            correlation ID: 994d5b05-7775-48cd-93aa-f17d4c56d0ef\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-jxwc9m\" for cluster \"cluster-candidate-5-1-jxwc9m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2089be556480\u003e: \n    failed to create NodePool np-jxwc9m: failed waiting for nodepool=\"np-jxwc9m\" for cluster \"cluster-candidate-5-1-jxwc9m\" in resourcegroup=\"rg-candidate-5-1-t5bnkh9ptx8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\",\n      \"name\": \"aa13a852-544e-4ae1-9ee9-db477af0802a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:36:20.928028726Z\",\n      \"endTime\": \"2026-10-27T08:55:28.505111283Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-nmkn7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-rxvtq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-nmkn7: Creating: virtualmachine creating or updating\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-rxvtq: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\n            correlation ID: 994d5b05-7775-48cd-93aa-f17d4c56d0ef\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:96]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1520efb25aa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-g5mddw2znmwk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\",\n      \"name\": \"a9c68b69-2f13-4771-88f2-779ee3c74191\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:30:17.391270547Z\",\n      \"endTime\": \"2026-10-27T08:49:18.18634688Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-gzw7k-gkpzc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v20261001-np-1-gzw7k-sxqb6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-gzw7k-gkpzc: Creating: virtualmachine creating or updating\\nMachine v20261001-np-1-gzw7k-sxqb6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\n            correlation ID: c7660621-545a-4ad8-89c0-9dd90d63841c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:96]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1520efb25aa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-g5mddw2znmwk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\",\n      \"name\": \"a9c68b69-2f13-4771-88f2-779ee3c74191\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:30:17.391270547Z\",\n      \"endTime\": \"2026-10-27T08:49:18.18634688Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-gzw7k-gkpzc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v20261001-np-1-gzw7k-sxqb6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-gzw7k-gkpzc: Creating: virtualmachine creating or updating\\nMachine v20261001-np-1-gzw7k-sxqb6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\n            correlation ID: c7660621-545a-4ad8-89c0-9dd90d63841c\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3baee6990108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2gdwf4pbz22s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\",\n      \"name\": \"0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:12:10.541346261Z\",\n      \"endTime\": \"2026-10-27T08:31:14.925265024Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-ffmd5-h94sn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-ffmd5-hvcp5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-ffmd5-h94sn: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-ffmd5-hvcp5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\n            correlation ID: 6e36e413-6dec-4393-bcb3-4788f0d108b3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3baee6990108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2gdwf4pbz22s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\",\n      \"name\": \"0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:12:10.541346261Z\",\n      \"endTime\": \"2026-10-27T08:31:14.925265024Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-ffmd5-h94sn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-ffmd5-hvcp5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-ffmd5-h94sn: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-ffmd5-hvcp5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\n            correlation ID: 6e36e413-6dec-4393-bcb3-4788f0d108b3\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"13447a28327f6cac72eeb38ac4f64daeca2de13dce1a8779d5b023237c94cc0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"48f521861c358da162e56adb1d5a5560a6d110197a1e1ace87d26c0bd91ae752","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"5bf62fc718ddab8012057c407f628773cdec5fa1f4dce294092b41dd220fca46","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"5d0bce7e827e2d05c2cc18c9afe55ae2811c60bd7230edf27e23c249ad468b73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"65786b9fd4ff52613cb0353320693acaf1aa58e9e4797f497abddc9ba6d5c20f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"7a2b3613d8cecf72125933f4f03d2d7d31c60d9319bc6e32bde74818fbf4a7c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"80f41f56d6bb15b31124a77cca8ab3bafdbb09260df5d6a0be927f35414248b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"aac52a12f1e11813e645caf10fcf25877853fed7ef788c7c5c5971b57aebcaa4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"bdeacb1eac1daccd351ab2768ef041b978c2eb7deaf336006af84dd3f773eae0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"d84676500a8d1f8f5aba72d56f9519c4ab3fe5ba3beacd4c258d168261c4440e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"e7eacd17b93a9922f30449fc8ac0b2b6c2f322360a8ba998276c9e141732d710","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":11.764705882352942,"anchor_occurrences":5,"after_last_push_count":28,"also_in":["PROD"],"trend_counts":[0,2,3,0,0,0,0,0],"trend_range":"2026-10-25..2026-11-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-24T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x66df03e8b20\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x66df03e8b20\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2de6fa3020f0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2de6fa3020f0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:260]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-23-wlxxtl\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x30c9514302d0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:260]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-23-wlxxtl\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x30c9514302d0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","occurred_at":"2026-10-27T02:00:05Z","signature_id":"5f7305d0b3b91c61eb8e4f56e572507a452aa85cc00ec307edf3b8e9bfddebb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","occurred_at":"2026-10-27T02:00:05Z","signature_id":"70755a49efebe5ef08cdb2d0691a83edd6fc97c800ce0e052ac99acb90b8d7f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","occurred_at":"2026-10-27T02:00:05Z","signature_id":"e6435ae321fc270025d85586172e832697f963663729776cb49988ed3cb99186","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2114854607929741312","occurred_at":"2026-10-26T23:00:05Z","signature_id":"c3805a868ccf40727d210350cbe8cf8e87890bfe820ceae256095c60d80a32e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2114854607929741312","occurred_at":"2026-10-26T23:00:05Z","signature_id":"f974db37c7eb3d9bbc97893012fdea83ddca24ef2f512221011480246752904a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ff366311fd43d72666297c0085142c8518628a4849f37fc8ec0089c5423ea5c2","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":11.764705882352942,"anchor_occurrences":3,"after_last_push_count":20,"also_in":["PROD"],"trend_counts":[0,2,1,0,0,0,0,0],"trend_range":"2026-10-25..2026-11-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-24T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-2nz4xc\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x203abb848000\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x203ab927a940\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-2nz4xc\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x203abb848000\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x203ab927a940\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-5-0-q6mvvg\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1ec67b8dad40\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x1ec67cfb92a0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-5-0-q6mvvg\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1ec67b8dad40\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x1ec67cfb92a0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-9qvdk7\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x38ab336f65a0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x38ab34e123c0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-9qvdk7\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x38ab336f65a0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x38ab34e123c0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","occurred_at":"2026-10-27T02:00:05Z","signature_id":"cb8a33e2b2095072faf50f6e0575b49f9f24cba8d6e5696426aaf71bfa169c8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2114854607929741312","occurred_at":"2026-10-26T23:00:05Z","signature_id":"7254a4758a4c9f3547d8595bb271022e57f6d2108cbb99621ddf2051d8a76f43","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2114854607929741312","occurred_at":"2026-10-26T23:00:05Z","signature_id":"f6440923d584406c047bf35485df58b43af2e7dc6ec14ef4aa1c454df23ab759","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"93bb2520ef0f34dd66efce84b14b6252dd8c25360081fb21aba7a8fa22f714a9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV","INT","PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-25..2026-11-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-22T02:00:10Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-vrx699\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb37c723e620\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-vrx699: failed waiting for cluster=\"cluster-zstream-4-22-vrx699\" in resourcegroup=\"rg-zstream-upgrade-4-22-mk5975tqzhdb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\",\n      \"name\": \"f522dee9-4911-4a39-a8b7-66d53ae810aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T21:21:44.905508951Z\",\n      \"endTime\": \"2026-10-28T21:50:49.351811692Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\n            correlation ID: fa068c44-117e-4214-8c6d-206698ccb0ad\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-vrx699\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb37c723e620\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-vrx699: failed waiting for cluster=\"cluster-zstream-4-22-vrx699\" in resourcegroup=\"rg-zstream-upgrade-4-22-mk5975tqzhdb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\",\n      \"name\": \"f522dee9-4911-4a39-a8b7-66d53ae810aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T21:21:44.905508951Z\",\n      \"endTime\": \"2026-10-28T21:50:49.351811692Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\n            correlation ID: fa068c44-117e-4214-8c6d-206698ccb0ad\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2115540873000259584","occurred_at":"2026-10-28T20:27:03Z","signature_id":"851111d98b1137aabdf43323fadae9cc3cc706ab194c4673dfabf5d420922313","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0ece5aa7e8b4a7df4eadead8c6c087ae24930024b2ab923cda14a7ad621f8a51","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-25..2026-11-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-03T00:25:08Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xa0fb7d04138\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-xf9jw4l5cdpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\",\n      \"name\": \"0006ad13-87f3-4fc4-8b43-20306abddf58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:14:54.971524126Z\",\n      \"endTime\": \"2026-10-27T08:33:58.162558969Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-7cl7p-8fh5m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-7cl7p-fd22m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-7cl7p-8fh5m: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-7cl7p-fd22m: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\n            correlation ID: 7bfa7cc1-239c-4ae3-a9bc-578e94ecc5a3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xa0fb7d04138\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-xf9jw4l5cdpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\",\n      \"name\": \"0006ad13-87f3-4fc4-8b43-20306abddf58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:14:54.971524126Z\",\n      \"endTime\": \"2026-10-27T08:33:58.162558969Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-7cl7p-8fh5m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-7cl7p-fd22m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-7cl7p-8fh5m: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-7cl7p-fd22m: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\n            correlation ID: 7bfa7cc1-239c-4ae3-a9bc-578e94ecc5a3\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"2a7dcb6ff7dcda026460bada2d10af749d2aaf152cdd767259012c980422e2c3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"96eb8165e1517c94f94344a079943853da38a18f87e198ac97e2ca5f84f1f2e3","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-25..2026-11-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-12T23:00:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:168]: failed to verify HCP cluster \"cluster-nightly-4-21-htnxwk\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x28946bc08d20\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x28946b722320\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x28946b722300\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x28946b314d20\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x28946b60da40\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x28946b60da20\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x28946b518dc0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:168]: failed to verify HCP cluster \"cluster-nightly-4-21-htnxwk\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x28946bc08d20\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x28946b722320\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x28946b722300\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x28946b314d20\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x28946b60da40\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x28946b60da20\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x28946b518dc0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2115217181124857856","occurred_at":"2026-10-27T23:00:49Z","signature_id":"2081f477b523b7866b3f60de53b0fb6ab5d90668ea1f7f412ab3ed026d28073d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c349dea01a43efbfe8bada3f00e6006e559aa06c48ac45e43b919b7539b25770","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-25..2026-11-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-23T02:00:29Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:510]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3b09d8d2ba10\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:510]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3b09d8d2ba10\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2115375462514429952","occurred_at":"2026-10-28T09:29:46Z","signature_id":"bfb9b703f5241dd736b9902336860998ffe6bdf4ba01f9a3a8b4d81786acbd9e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-d4wglkkdjhfj\" to finish creating: GET https://management","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-25..2026-11-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":28,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x8979cd42dc0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-d4wglkkdjhfj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\",\n      \"name\": \"4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T03:01:21.21908488Z\",\n      \"endTime\": \"2026-10-28T03:20:21.275752007Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p0i5l3o3i9z8d7r-np-deletable-95t5c-lvd52: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p0i5l3o3i9z8d7r-np-deletable-95t5c-lvd52: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\n            correlation ID: 48b19522-cae3-4702-8aae-982e9a4042a0\n            client request ID: unavailable\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x8979cd42dc0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-d4wglkkdjhfj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\",\n      \"name\": \"4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T03:01:21.21908488Z\",\n      \"endTime\": \"2026-10-28T03:20:21.275752007Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p0i5l3o3i9z8d7r-np-deletable-95t5c-lvd52: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p0i5l3o3i9z8d7r-np-deletable-95t5c-lvd52: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\n            correlation ID: 48b19522-cae3-4702-8aae-982e9a4042a0\n            client request ID: unavailable"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2115262480291729408","occurred_at":"2026-10-28T02:00:49Z","signature_id":"a130358adaef86ab6bdbcc865713e8b92da4c632acc5d4fbbe39da19d0be4f65","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3e7244a85da8bd67de02bd0b8b84b756179c64ce2823e309e63b548314c55c8e","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4j5vtdb84h8v\" to finish creating: GET https://management","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-25..2026-11-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":28,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3b12e2cba60\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4j5vtdb84h8v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\",\n      \"name\": \"083c3fe4-0c43-4b6a-9905-e8c1aa97e192\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T09:13:51.032005174Z\",\n      \"endTime\": \"2026-10-27T09:32:53.54866754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-nd4jp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-q624c: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-nd4jp: Creating: virtualmachine creating or updating\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-q624c: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\n            correlation ID: 01ff1c80-ae7b-4cd9-aeb7-68197786fff9\n            client request ID: unavailable\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3b12e2cba60\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4j5vtdb84h8v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\",\n      \"name\": \"083c3fe4-0c43-4b6a-9905-e8c1aa97e192\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T09:13:51.032005174Z\",\n      \"endTime\": \"2026-10-27T09:32:53.54866754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-nd4jp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-q624c: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-nd4jp: Creating: virtualmachine creating or updating\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-q624c: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\n            correlation ID: 01ff1c80-ae7b-4cd9-aeb7-68197786fff9\n            client request ID: unavailable"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"b9c3c5012ff77c3681e350e06c44d7277c66955d064a006d326eb01c4d066632","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}