{"meta":{"start_date":"2026-09-29","end_date":"2026-09-29","start_at":"2026-09-29T00:00:00Z","end_at":"2026-09-30T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T10:00:53Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":4,"raw_occurrences":7,"matched_occurrences":7,"runs_affected":4},"rows":[{"environment":"int","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":50,"anchor_occurrences":4,"after_last_push_count":6,"trend_counts":[0,0,0,0,1,1,4],"trend_range":"2026-09-23..2026-09-29 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-27T07:04:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00134a240\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lf58x6mrt8sn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6899b842-6be7-4752-9378-45ead2f46f38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6899b842-6be7-4752-9378-45ead2f46f38\",\n      \"name\": \"6899b842-6be7-4752-9378-45ead2f46f38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T22:48:49.695468888Z\",\n      \"endTime\": \"2026-09-29T23:07:57.514241775Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-lf58x6mrt8sn--managed/providers/Microsoft.Compute/virtualMachines/j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-lf58x6mrt8sn--managed/providers/Microsoft.Compute/virtualMachines/j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00134a240\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lf58x6mrt8sn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6899b842-6be7-4752-9378-45ead2f46f38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6899b842-6be7-4752-9378-45ead2f46f38\",\n      \"name\": \"6899b842-6be7-4752-9378-45ead2f46f38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T22:48:49.695468888Z\",\n      \"endTime\": \"2026-09-29T23:07:57.514241775Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-lf58x6mrt8sn--managed/providers/Microsoft.Compute/virtualMachines/j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-lf58x6mrt8sn--managed/providers/Microsoft.Compute/virtualMachines/j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010311c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-zbbkqjx7fwrv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a80b6b-fe14-4fa6-a826-e605868e189f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a80b6b-fe14-4fa6-a826-e605868e189f\",\n      \"name\": \"15a80b6b-fe14-4fa6-a826-e605868e189f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T21:44:03.296557Z\",\n      \"endTime\": \"2026-09-29T22:03:11.679917905Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-kclwz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-kclwz to exist\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-kclwz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-kclwz to exist\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010311c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-zbbkqjx7fwrv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a80b6b-fe14-4fa6-a826-e605868e189f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a80b6b-fe14-4fa6-a826-e605868e189f\",\n      \"name\": \"15a80b6b-fe14-4fa6-a826-e605868e189f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T21:44:03.296557Z\",\n      \"endTime\": \"2026-09-29T22:03:11.679917905Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-kclwz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-kclwz to exist\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-kclwz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-kclwz to exist\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f32080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-thvwrhnx9f2p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa62eeea-3561-4f5f-872a-af727c8e3ec4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa62eeea-3561-4f5f-872a-af727c8e3ec4\",\n      \"name\": \"aa62eeea-3561-4f5f-872a-af727c8e3ec4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T16:37:13.215287875Z\",\n      \"endTime\": \"2026-09-29T16:56:16.225730373Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5 to exist\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5 to exist\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f32080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-thvwrhnx9f2p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa62eeea-3561-4f5f-872a-af727c8e3ec4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa62eeea-3561-4f5f-872a-af727c8e3ec4\",\n      \"name\": \"aa62eeea-3561-4f5f-872a-af727c8e3ec4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T16:37:13.215287875Z\",\n      \"endTime\": \"2026-09-29T16:56:16.225730373Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5 to exist\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5 to exist\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105040832624070656","occurred_at":"2026-09-29T21:03:38Z","signature_id":"3b25e49ebcf01ceefe5ef6d15772a6b3207794a1d640c359ac345895674ed15c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105040832624070656","occurred_at":"2026-09-29T21:03:38Z","signature_id":"607e94a35a81473d478091f2da267ba95cac66f639d687416633136e0271f1e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104966711995273216","occurred_at":"2026-09-29T16:09:07Z","signature_id":"464235197a1662fe879ebc18d73f5450d313eafaf40b203a076ea03d5c5ec2ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104927790917750784","occurred_at":"2026-09-29T13:34:27Z","signature_id":"b47ba10dff74b3c276896ac768dedd60fa50be8640a5fc7175dfae6a540b5f0c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":50,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-09-23..2026-09-29 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00143c978\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00143c978\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00090a4c8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00090a4c8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00061eb40\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00061eb40\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104966711995273216","occurred_at":"2026-09-29T16:09:07Z","signature_id":"3f1b5d3d9c68310f2e07573951e1f580c91e75043ece8b8cef2bcc6b19f1f6b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104927790917750784","occurred_at":"2026-09-29T13:34:27Z","signature_id":"90bf9bad3db34ef7342fcb200f948ba1cd05e545d40dcef95246cf50da7d0cc4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104877329665757184","occurred_at":"2026-09-29T10:13:56Z","signature_id":"cb0b865ce23da66a9ce61d2cfe51d7b9448f029a36bab5dde0e72f51033a5d6f","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}