{"meta":{"date":"2026-09-25","timezone":"UTC","generated_at":"2026-10-30T03:44:40Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":7,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103598326791278592","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T21:31:38Z","started_at":"2026-09-25T21:31:38Z","completed_at":"2026-09-25T23:52:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T00:12:25.481026831Z","timing_metadata_checked_at":"2026-09-26T00:12:25.481026831Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T00:12:25.485814316Z","region_metadata_checked_at":"2026-09-26T00:12:25.485814316Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103550178567917568","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T18:20:19Z","started_at":"2026-09-25T18:20:19Z","completed_at":"2026-09-25T20:39:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:52:34.415166804Z","timing_metadata_checked_at":"2026-09-25T20:52:34.415166804Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:52:34.417962396Z","region_metadata_checked_at":"2026-09-25T20:52:34.417962396Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103500347635929088","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T15:01:22Z","started_at":"2026-09-25T15:01:22Z","completed_at":"2026-09-25T17:18:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T17:34:03.455730885Z","timing_metadata_checked_at":"2026-09-25T17:34:03.455730885Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T17:34:03.459086976Z","region_metadata_checked_at":"2026-09-25T17:34:03.459086976Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"40aa2ba53f2aef741b0ce0aaf893b24065b7b4b63550e8552950e011b0263ed3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103500347635929088","occurred_at":"2026-09-25T15:01:22Z","failed_at":"e2e","signature_id":"2374477c920f24f39cbd859778b4b5423d9aa24bc3495ca463c798a21c8c8e87","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000599de0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-kcj4l5\" in resourcegroup=\"rg-np-skip-minor-kcj4l5-45s8nhm4k78b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da564fd3-aea3-4a41-b171-3969372d023d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da564fd3-aea3-4a41-b171-3969372d023d\",\n      \"name\": \"da564fd3-aea3-4a41-b171-3969372d023d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T16:39:04.653915252Z\",\n      \"endTime\": \"2026-09-25T16:58:09.716908828Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kcj4l5-45s8nhm4k78b-np-sm-kcj4l5--managed/providers/Microsoft.Compute/virtualMachines/k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kcj4l5-45s8nhm4k78b-np-sm-kcj4l5--managed/providers/Microsoft.Compute/virtualMachines/k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000599de0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-kcj4l5\" in resourcegroup=\"rg-np-skip-minor-kcj4l5-45s8nhm4k78b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da564fd3-aea3-4a41-b171-3969372d023d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da564fd3-aea3-4a41-b171-3969372d023d\",\n      \"name\": \"da564fd3-aea3-4a41-b171-3969372d023d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T16:39:04.653915252Z\",\n      \"endTime\": \"2026-09-25T16:58:09.716908828Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kcj4l5-45s8nhm4k78b-np-sm-kcj4l5--managed/providers/Microsoft.Compute/virtualMachines/k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kcj4l5-45s8nhm4k78b-np-sm-kcj4l5--managed/providers/Microsoft.Compute/virtualMachines/k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bfd08cdd46bd2b7366c25bc24686fba64cd529977b75f42a98cbe09926d75184","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103500347635929088","occurred_at":"2026-09-25T15:01:22Z","failed_at":"e2e","signature_id":"e1f4ee9af599a77b687540e9f4697061e341edc13f3416df00ca7e8b696944ff","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18e80\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-p9kv2wz66ng2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d3308e-222c-40d8-a1be-8c057c26bab1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d3308e-222c-40d8-a1be-8c057c26bab1\",\n      \"name\": \"82d3308e-222c-40d8-a1be-8c057c26bab1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T15:45:43.486812846Z\",\n      \"endTime\": \"2026-09-25T16:14:49.560161271Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18e80\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-p9kv2wz66ng2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d3308e-222c-40d8-a1be-8c057c26bab1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d3308e-222c-40d8-a1be-8c057c26bab1\",\n      \"name\": \"82d3308e-222c-40d8-a1be-8c057c26bab1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T15:45:43.486812846Z\",\n      \"endTime\": \"2026-09-25T16:14:49.560161271Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"53d6a9f6efeca45a9da0e1f8eb60e93e70de646c11d05978730da3382fb8e964","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254","53d6a9f6efeca45a9da0e1f8eb60e93e70de646c11d05978730da3382fb8e964"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103448928220352512","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T11:37:59Z","started_at":"2026-09-25T11:37:59Z","completed_at":"2026-09-25T13:59:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T14:12:39.754358513Z","timing_metadata_checked_at":"2026-09-25T14:12:39.754358513Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T14:12:39.757426004Z","region_metadata_checked_at":"2026-09-25T14:12:39.757426004Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103397522356572160","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T08:13:43Z","started_at":"2026-09-25T08:13:43Z","completed_at":"2026-09-25T10:18:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T10:30:30.754722619Z","timing_metadata_checked_at":"2026-09-25T10:30:30.754722619Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T10:30:30.758208008Z","region_metadata_checked_at":"2026-09-25T10:30:30.758208008Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9c53ba9a71bdfa205b67e4c612156308ce86dd9ed10bf70b233e8635f01db0e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103397522356572160","occurred_at":"2026-09-25T08:13:43Z","failed_at":"e2e","signature_id":"4b38b160a22968120417e984b979b00dd8bb78826d4b38714c0eb589ea58bd2f","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00075fc80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-dnhw57q2zhsr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\",\n      \"name\": \"9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T08:30:53.138668141Z\",\n      \"endTime\": \"2026-09-25T08:49:53.966422643Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-dnhw57q2zhsr--managed/providers/Microsoft.Compute/virtualMachines/q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-dnhw57q2zhsr--managed/providers/Microsoft.Compute/virtualMachines/q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00075fc80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-dnhw57q2zhsr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\",\n      \"name\": \"9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T08:30:53.138668141Z\",\n      \"endTime\": \"2026-09-25T08:49:53.966422643Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-dnhw57q2zhsr--managed/providers/Microsoft.Compute/virtualMachines/q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-dnhw57q2zhsr--managed/providers/Microsoft.Compute/virtualMachines/q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cd3b62302145a4f193634c04292cd7c3b0522163606e6fe1f23b2a097a00d90e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103397522356572160","occurred_at":"2026-09-25T08:13:43Z","failed_at":"e2e","signature_id":"b009af289e732a1b6a4d96b5624455888514f82524439fc4e668740147e9e47a","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c02c0\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2mw49n77wnb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e57de6e4-350f-44ca-8989-0fa815b3fd2f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e57de6e4-350f-44ca-8989-0fa815b3fd2f\",\n      \"name\": \"e57de6e4-350f-44ca-8989-0fa815b3fd2f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T08:27:10.94071903Z\",\n      \"endTime\": \"2026-09-25T08:46:13.959695236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-2mw49n77wnb2--managed/providers/Microsoft.Compute/virtualMachines/y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-2mw49n77wnb2--managed/providers/Microsoft.Compute/virtualMachines/y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c02c0\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2mw49n77wnb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e57de6e4-350f-44ca-8989-0fa815b3fd2f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e57de6e4-350f-44ca-8989-0fa815b3fd2f\",\n      \"name\": \"e57de6e4-350f-44ca-8989-0fa815b3fd2f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T08:27:10.94071903Z\",\n      \"endTime\": \"2026-09-25T08:46:13.959695236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-2mw49n77wnb2--managed/providers/Microsoft.Compute/virtualMachines/y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-2mw49n77wnb2--managed/providers/Microsoft.Compute/virtualMachines/y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103361291237199872","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T05:49:44Z","started_at":"2026-09-25T05:49:44Z","completed_at":"2026-09-25T08:08:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T08:18:47.51872031Z","timing_metadata_checked_at":"2026-09-25T08:18:47.51872031Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T08:18:47.521194301Z","region_metadata_checked_at":"2026-09-25T08:18:47.521194301Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7e2edfcbfb6ac03ecec6c3f12d0711e76729f99e0c71eacee7353ad38a8bcd93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103361291237199872","occurred_at":"2026-09-25T05:49:44Z","failed_at":"e2e","signature_id":"26173be16e73b2272c2dda84a4b010bd8ef49864c818286cf90989c383c22b93","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pt5qmx\" for cluster \"cluster-candidate-5-0-pt5qmx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c10900\u003e: \n    failed to create NodePool np-pt5qmx: failed starting nodepool creation \"np-pt5qmx\" for cluster \"cluster-candidate-5-0-pt5qmx\" in resourcegroup=\"rg-candidate-5-0-9clwz8rf6599\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-9clwz8rf6599/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-pt5qmx/nodePools/np-pt5qmx\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"provisioning\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-9clwz8rf6599/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-pt5qmx/nodePools/np-pt5qmx\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pt5qmx\" for cluster \"cluster-candidate-5-0-pt5qmx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c10900\u003e: \n    failed to create NodePool np-pt5qmx: failed starting nodepool creation \"np-pt5qmx\" for cluster \"cluster-candidate-5-0-pt5qmx\" in resourcegroup=\"rg-candidate-5-0-9clwz8rf6599\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-9clwz8rf6599/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-pt5qmx/nodePools/np-pt5qmx\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"provisioning\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-9clwz8rf6599/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-pt5qmx/nodePools/np-pt5qmx\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"46b0598b5ebe8f06dc5f7cfd83233a53219d8dfdc2ad61e72855646ca0260a66","failure_pattern":"ERROR CODE: Conflict; detail message Cannot update resource while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["46b0598b5ebe8f06dc5f7cfd83233a53219d8dfdc2ad61e72855646ca0260a66"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103303905172525056","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T02:01:43Z","started_at":"2026-09-25T02:01:43Z","completed_at":"2026-09-25T04:24:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T04:29:56.340156911Z","timing_metadata_checked_at":"2026-09-25T04:29:56.340156911Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T04:29:56.343166302Z","region_metadata_checked_at":"2026-09-25T04:29:56.343166302Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1d81a42ca2463d1175410f06c66301700ab11f4ca2717cf06cc6629ebc8e4d1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103303905172525056","occurred_at":"2026-09-25T02:01:43Z","failed_at":"e2e","signature_id":"f35b530f26f0d3fedd585cc4516c1a10ecb44588c174c5fe5a05614b69732b79","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007453e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-58b5tglh77bz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28fe561e-2b93-4934-b274-0e50cb5390d6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28fe561e-2b93-4934-b274-0e50cb5390d6\",\n      \"name\": \"28fe561e-2b93-4934-b274-0e50cb5390d6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T02:36:40.600765804Z\",\n      \"endTime\": \"2026-09-25T02:55:41.450889323Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-58b5tglh77bz--managed/providers/Microsoft.Compute/virtualMachines/x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-58b5tglh77bz--managed/providers/Microsoft.Compute/virtualMachines/x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007453e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-58b5tglh77bz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28fe561e-2b93-4934-b274-0e50cb5390d6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28fe561e-2b93-4934-b274-0e50cb5390d6\",\n      \"name\": \"28fe561e-2b93-4934-b274-0e50cb5390d6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T02:36:40.600765804Z\",\n      \"endTime\": \"2026-09-25T02:55:41.450889323Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-58b5tglh77bz--managed/providers/Microsoft.Compute/virtualMachines/x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-58b5tglh77bz--managed/providers/Microsoft.Compute/virtualMachines/x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}