{"meta":{"date":"2026-09-29","timezone":"UTC","generated_at":"2026-10-29T23:32:54Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105040832624070656","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T21:03:38Z","started_at":"2026-09-29T21:03:38Z","completed_at":"2026-09-29T23:41:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T23:51:07.306596512Z","timing_metadata_checked_at":"2026-09-29T23:51:07.306596512Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T23:51:07.309745102Z","region_metadata_checked_at":"2026-09-29T23:51:07.309745102Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"85bd1f835147fc2021f054a8bf27e3fefe6b32579972fe7fcd9d6c70909fae4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105040832624070656","occurred_at":"2026-09-29T21:03:38Z","failed_at":"e2e","signature_id":"3b25e49ebcf01ceefe5ef6d15772a6b3207794a1d640c359ac345895674ed15c","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00134a240\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lf58x6mrt8sn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6899b842-6be7-4752-9378-45ead2f46f38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6899b842-6be7-4752-9378-45ead2f46f38\",\n      \"name\": \"6899b842-6be7-4752-9378-45ead2f46f38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T22:48:49.695468888Z\",\n      \"endTime\": \"2026-09-29T23:07:57.514241775Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-lf58x6mrt8sn--managed/providers/Microsoft.Compute/virtualMachines/j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-lf58x6mrt8sn--managed/providers/Microsoft.Compute/virtualMachines/j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00134a240\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lf58x6mrt8sn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6899b842-6be7-4752-9378-45ead2f46f38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6899b842-6be7-4752-9378-45ead2f46f38\",\n      \"name\": \"6899b842-6be7-4752-9378-45ead2f46f38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T22:48:49.695468888Z\",\n      \"endTime\": \"2026-09-29T23:07:57.514241775Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-lf58x6mrt8sn--managed/providers/Microsoft.Compute/virtualMachines/j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-lf58x6mrt8sn--managed/providers/Microsoft.Compute/virtualMachines/j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e5568b11d6d0902b98c685a3c0510e57a63a12ce81a618b3ddbec2eedc7730b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105040832624070656","occurred_at":"2026-09-29T21:03:38Z","failed_at":"e2e","signature_id":"607e94a35a81473d478091f2da267ba95cac66f639d687416633136e0271f1e0","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010311c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-zbbkqjx7fwrv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a80b6b-fe14-4fa6-a826-e605868e189f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a80b6b-fe14-4fa6-a826-e605868e189f\",\n      \"name\": \"15a80b6b-fe14-4fa6-a826-e605868e189f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T21:44:03.296557Z\",\n      \"endTime\": \"2026-09-29T22:03:11.679917905Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-kclwz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-kclwz to exist\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-kclwz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-kclwz to exist\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010311c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-zbbkqjx7fwrv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a80b6b-fe14-4fa6-a826-e605868e189f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a80b6b-fe14-4fa6-a826-e605868e189f\",\n      \"name\": \"15a80b6b-fe14-4fa6-a826-e605868e189f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T21:44:03.296557Z\",\n      \"endTime\": \"2026-09-29T22:03:11.679917905Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-kclwz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-kclwz to exist\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-kclwz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-kclwz to exist\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104966711995273216","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T16:09:07Z","started_at":"2026-09-29T16:09:07Z","completed_at":"2026-09-29T18:35:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T18:36:50.996686342Z","timing_metadata_checked_at":"2026-09-29T18:36:50.996686342Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T18:36:51.002413324Z","region_metadata_checked_at":"2026-09-29T18:36:51.002413324Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"701e26cd2f9735fcabd7e2e1780b1057d200ccd5ae566683c41e046efd1df191","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104966711995273216","occurred_at":"2026-09-29T16:09:07Z","failed_at":"e2e","signature_id":"464235197a1662fe879ebc18d73f5450d313eafaf40b203a076ea03d5c5ec2ee","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f32080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-thvwrhnx9f2p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa62eeea-3561-4f5f-872a-af727c8e3ec4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa62eeea-3561-4f5f-872a-af727c8e3ec4\",\n      \"name\": \"aa62eeea-3561-4f5f-872a-af727c8e3ec4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T16:37:13.215287875Z\",\n      \"endTime\": \"2026-09-29T16:56:16.225730373Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5 to exist\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5 to exist\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f32080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-thvwrhnx9f2p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa62eeea-3561-4f5f-872a-af727c8e3ec4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa62eeea-3561-4f5f-872a-af727c8e3ec4\",\n      \"name\": \"aa62eeea-3561-4f5f-872a-af727c8e3ec4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T16:37:13.215287875Z\",\n      \"endTime\": \"2026-09-29T16:56:16.225730373Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5 to exist\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5 to exist\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d94d41abb3b7734e8c90a8fbe43ef5a58bb24dd70087bb3075a473a7b7242d3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104966711995273216","occurred_at":"2026-09-29T16:09:07Z","failed_at":"e2e","signature_id":"3f1b5d3d9c68310f2e07573951e1f580c91e75043ece8b8cef2bcc6b19f1f6b8","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00143c978\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00143c978\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104927790917750784","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T13:34:27Z","started_at":"2026-09-29T13:34:27Z","completed_at":"2026-09-29T16:03:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T16:10:32.61749758Z","timing_metadata_checked_at":"2026-09-29T16:10:32.61749758Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T16:10:32.622228764Z","region_metadata_checked_at":"2026-09-29T16:10:32.622228764Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"45a34f13f41fded10d774e60d54af8c83ac821652e84e79576fe99ee8ad005b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104927790917750784","occurred_at":"2026-09-29T13:34:27Z","failed_at":"e2e","signature_id":"90bf9bad3db34ef7342fcb200f948ba1cd05e545d40dcef95246cf50da7d0cc4","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00090a4c8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00090a4c8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"c35162e5bd75a32a4e6a54be18f2624b1b28d4d7e0251835d9388582d1b6b1a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104927790917750784","occurred_at":"2026-09-29T13:34:27Z","failed_at":"e2e","signature_id":"b47ba10dff74b3c276896ac768dedd60fa50be8640a5fc7175dfae6a540b5f0c","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009bd5e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-2njpnbrz5jg5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e034572f-71f9-4484-a06b-08d411aba3d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e034572f-71f9-4484-a06b-08d411aba3d9\",\n      \"name\": \"e034572f-71f9-4484-a06b-08d411aba3d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T14:28:57.448756437Z\",\n      \"endTime\": \"2026-09-29T14:48:03.628370174Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n1q8y3e1s2j2m9q-np-1-spldz-2nngd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-2njpnbrz5jg5--managed/providers/Microsoft.Compute/virtualMachines/n1q8y3e1s2j2m9q-np-1-spldz-2nngd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n1q8y3e1s2j2m9q-np-1-spldz-2nngd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-2njpnbrz5jg5--managed/providers/Microsoft.Compute/virtualMachines/n1q8y3e1s2j2m9q-np-1-spldz-2nngd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009bd5e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-2njpnbrz5jg5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e034572f-71f9-4484-a06b-08d411aba3d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e034572f-71f9-4484-a06b-08d411aba3d9\",\n      \"name\": \"e034572f-71f9-4484-a06b-08d411aba3d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T14:28:57.448756437Z\",\n      \"endTime\": \"2026-09-29T14:48:03.628370174Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n1q8y3e1s2j2m9q-np-1-spldz-2nngd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-2njpnbrz5jg5--managed/providers/Microsoft.Compute/virtualMachines/n1q8y3e1s2j2m9q-np-1-spldz-2nngd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n1q8y3e1s2j2m9q-np-1-spldz-2nngd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-2njpnbrz5jg5--managed/providers/Microsoft.Compute/virtualMachines/n1q8y3e1s2j2m9q-np-1-spldz-2nngd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104877329665757184","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T10:13:56Z","started_at":"2026-09-29T10:13:56Z","completed_at":"2026-09-29T12:36:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T12:50:34.590538421Z","timing_metadata_checked_at":"2026-09-29T12:50:34.590538421Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T12:50:34.596649409Z","region_metadata_checked_at":"2026-09-29T12:50:34.596649409Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"dd86161a5ecafec54a4670bc3ba3e5a4be04874ccba82a952038f7aacc338e14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104877329665757184","occurred_at":"2026-09-29T10:13:56Z","failed_at":"e2e","signature_id":"cb0b865ce23da66a9ce61d2cfe51d7b9448f029a36bab5dde0e72f51033a5d6f","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00061eb40\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00061eb40\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104823044923985920","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T06:38:14Z","started_at":"2026-09-29T06:38:14Z","completed_at":"2026-09-29T08:50:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T09:00:37.097647876Z","timing_metadata_checked_at":"2026-09-29T09:00:37.097647876Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T09:00:37.100141268Z","region_metadata_checked_at":"2026-09-29T09:00:37.100141268Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104766693522804736","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T02:54:19Z","started_at":"2026-09-29T02:54:19Z","completed_at":"2026-09-29T05:14:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T05:29:31.113800237Z","timing_metadata_checked_at":"2026-09-29T05:29:31.113800237Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T05:29:31.117621725Z","region_metadata_checked_at":"2026-09-29T05:29:31.117621725Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}