{"meta":{"start_date":"2026-10-25","end_date":"2026-10-25","start_at":"2026-10-25T00:00:00Z","end_at":"2026-10-26T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T01:03:03Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":4,"raw_occurrences":6,"matched_occurrences":6,"runs_affected":4},"rows":[{"environment":"int","failure_pattern_id":"8b157df38af3252928b0d0c8ebb12f675589f14b62d8751da69093589eb2fca4","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,2,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:118]: failed to create default nodepool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3cdcfbca5680\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-8vtwfvfssx9k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0\",\n      \"name\": \"907ac069-8701-4021-b410-cf10181345c0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T05:14:31.336281353Z\",\n      \"endTime\": \"2026-10-25T05:33:33.578871475Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cluster-gpu-np-np-1-8v7xk-22mlx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-22mlx to exist\\nMachine cluster-gpu-np-np-1-8v7xk-p7hpl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-p7hpl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cluster-gpu-np-np-1-8v7xk-p7hpl: Failed: interfaces failed to create or update. err: failed to get existing resource rg-gpu-nodepool-8vtwfvfssx9k--managed/cluster-gpu-np-np-1-8v7xk-p7hpl-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Network/networkInterfaces/cluster-gpu-np-np-1-8v7xk-p7hpl-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to ... message truncated\\nMachine cluster-gpu-np-np-1-8v7xk-22mlx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-22mlx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0\n            correlation ID: 1c529a86-2bdd-43d7-ab0e-4267cd216305\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:118]: failed to create default nodepool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3cdcfbca5680\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-8vtwfvfssx9k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0\",\n      \"name\": \"907ac069-8701-4021-b410-cf10181345c0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T05:14:31.336281353Z\",\n      \"endTime\": \"2026-10-25T05:33:33.578871475Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cluster-gpu-np-np-1-8v7xk-22mlx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-22mlx to exist\\nMachine cluster-gpu-np-np-1-8v7xk-p7hpl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-p7hpl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cluster-gpu-np-np-1-8v7xk-p7hpl: Failed: interfaces failed to create or update. err: failed to get existing resource rg-gpu-nodepool-8vtwfvfssx9k--managed/cluster-gpu-np-np-1-8v7xk-p7hpl-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Network/networkInterfaces/cluster-gpu-np-np-1-8v7xk-p7hpl-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to ... message truncated\\nMachine cluster-gpu-np-np-1-8v7xk-22mlx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-22mlx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0\n            correlation ID: 1c529a86-2bdd-43d7-ab0e-4267cd216305\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114214308597469184","occurred_at":"2026-10-25T04:35:45Z","signature_id":"bd9e8e0cae4c284361718774eca639e561433edd1d045672abb6f55be42cb28b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"a943d2907074ed1f6f29dea5f81c6cf99bc3a7668efe0e4f83b02f9f453e38ff","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-c2hldx' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-g8n765\"\nUnexpected error:\n    \u003c*errors.joinError | 0x254aca410498\u003e: \n    VerifyIngressDNSRecord(aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.008s: DNS A record \"*.e2e-dns-c2hldx\" not yet present in zone rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/providers/Microsoft.Network/dnsZones/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-c2hldx\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-c2hldx' does not exist in resource group 'rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/providers/Microsoft.Network/dnsZones/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-c2hldx\n            correlation ID: 192cc89e-15d5-4e23-8532-45b3cbb07059\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-g8n765\"\nUnexpected error:\n    \u003c*errors.joinError | 0x254aca410498\u003e: \n    VerifyIngressDNSRecord(aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.008s: DNS A record \"*.e2e-dns-c2hldx\" not yet present in zone rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/providers/Microsoft.Network/dnsZones/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-c2hldx\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-c2hldx' does not exist in resource group 'rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/providers/Microsoft.Network/dnsZones/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-c2hldx\n            correlation ID: 192cc89e-15d5-4e23-8532-45b3cbb07059\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114253887920672768","occurred_at":"2026-10-25T07:13:02Z","signature_id":"93ba40c9e936f2e4832deda7d0989a60e6d92d450dd6a05b400d12ed625527fd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"1e4dfca010d42443afaf45bb97ca117493c5e3aa47ab2e1a79cdf35dff399c0c","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-jgd8x5' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-652kdm\"\nUnexpected error:\n    \u003c*errors.joinError | 0x282c0f144570\u003e: \n    VerifyIngressDNSRecord(aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.006s: DNS A record \"*.e2e-dns-jgd8x5\" not yet present in zone rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/providers/Microsoft.Network/dnsZones/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-jgd8x5\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-jgd8x5' does not exist in resource group 'rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/providers/Microsoft.Network/dnsZones/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-jgd8x5\n            correlation ID: a77d314a-799e-4560-8840-6738610cf0af\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-652kdm\"\nUnexpected error:\n    \u003c*errors.joinError | 0x282c0f144570\u003e: \n    VerifyIngressDNSRecord(aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.006s: DNS A record \"*.e2e-dns-jgd8x5\" not yet present in zone rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/providers/Microsoft.Network/dnsZones/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-jgd8x5\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-jgd8x5' does not exist in resource group 'rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/providers/Microsoft.Network/dnsZones/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-jgd8x5\n            correlation ID: a77d314a-799e-4560-8840-6738610cf0af\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114319204998975488","occurred_at":"2026-10-25T11:32:35Z","signature_id":"83028b765d97711c40f0cfa5de1d458e748cb143abe43b894f475efa5973d496","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"a969ecfcb2e0bc52f55dd728d038e5519456118b2d8e50c23ca4ff2b5d8d15cb","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-rg99mn' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-4fhvgp\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3188fb29e5a0\u003e: \n    VerifyIngressDNSRecord(aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.005s: DNS A record \"*.e2e-dns-rg99mn\" not yet present in zone rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/providers/Microsoft.Network/dnsZones/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-rg99mn\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-rg99mn' does not exist in resource group 'rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/providers/Microsoft.Network/dnsZones/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-rg99mn\n            correlation ID: cfd4563d-f15e-486d-ad09-c0f336987b6a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-4fhvgp\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3188fb29e5a0\u003e: \n    VerifyIngressDNSRecord(aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.005s: DNS A record \"*.e2e-dns-rg99mn\" not yet present in zone rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/providers/Microsoft.Network/dnsZones/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-rg99mn\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-rg99mn' does not exist in resource group 'rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/providers/Microsoft.Network/dnsZones/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-rg99mn\n            correlation ID: cfd4563d-f15e-486d-ad09-c0f336987b6a\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114344123895910400","occurred_at":"2026-10-25T13:11:36Z","signature_id":"e2f09681e2ed91758f8d8ecaa88afe2e6d3f1537b4d202fd8b86eb96323adc53","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"35f76f69751e6be795e750bc8688e506149495611433b9eeef712318ba1e0147","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x248e56e12588\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x248e56f14360\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x248e5757aa00\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x248e57018e60\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x248e56e12588\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x248e56f14360\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x248e5757aa00\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x248e57018e60\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114214308597469184","occurred_at":"2026-10-25T04:35:45Z","signature_id":"d67ff84dbdc57ce1c070c25b4715de2b3adc2b97c517e1ea3ab71e89571a78e8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"f8a1cfd7bf1a0bd0e82317121262b87bf51fd3aef00a088f1d465c1acea348e3","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-zhz5sxs2r4gt\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:657]: failed to get admin REST config\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xa204c386ab0\u003e: \n    failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-zhz5sxs2r4gt\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"acr-pull\\\" in resourcegroup=\\\"acr-pull-zhz5sxs2r4gt\\\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xa204b336550\u003e{\n                s: \"timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:657]: failed to get admin REST config\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xa204c386ab0\u003e: \n    failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-zhz5sxs2r4gt\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"acr-pull\\\" in resourcegroup=\\\"acr-pull-zhz5sxs2r4gt\\\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xa204b336550\u003e{\n                s: \"timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114253887920672768","occurred_at":"2026-10-25T07:13:02Z","signature_id":"3530be730cc90660f3f8d03767a868ff1c81d8a0d4db32dd7c61c6a501184ac1","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}