{"meta":{"date":"2026-10-28","timezone":"UTC","generated_at":"2026-10-28T23:57:28Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2115540873000259584","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T20:27:03Z","started_at":"2026-10-28T20:27:03Z","completed_at":"2026-10-28T22:50:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T22:59:42.331738131Z","timing_metadata_checked_at":"2026-10-28T22:59:42.331738131Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T22:59:42.33566282Z","region_metadata_checked_at":"2026-10-28T22:59:42.33566282Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5725e5508b5f7b6db6dbb4bfb3e6b510c4683a61ef024c1a96367763f663ebc5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2115540873000259584","occurred_at":"2026-10-28T20:27:03Z","failed_at":"unknown","signature_id":"851111d98b1137aabdf43323fadae9cc3cc706ab194c4673dfabf5d420922313","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-vrx699\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb37c723e620\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-vrx699: failed waiting for cluster=\"cluster-zstream-4-22-vrx699\" in resourcegroup=\"rg-zstream-upgrade-4-22-mk5975tqzhdb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\",\n      \"name\": \"f522dee9-4911-4a39-a8b7-66d53ae810aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T21:21:44.905508951Z\",\n      \"endTime\": \"2026-10-28T21:50:49.351811692Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\n            correlation ID: fa068c44-117e-4214-8c6d-206698ccb0ad\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-vrx699\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb37c723e620\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-vrx699: failed waiting for cluster=\"cluster-zstream-4-22-vrx699\" in resourcegroup=\"rg-zstream-upgrade-4-22-mk5975tqzhdb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\",\n      \"name\": \"f522dee9-4911-4a39-a8b7-66d53ae810aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T21:21:44.905508951Z\",\n      \"endTime\": \"2026-10-28T21:50:49.351811692Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\n            correlation ID: fa068c44-117e-4214-8c6d-206698ccb0ad\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93bb2520ef0f34dd66efce84b14b6252dd8c25360081fb21aba7a8fa22f714a9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93bb2520ef0f34dd66efce84b14b6252dd8c25360081fb21aba7a8fa22f714a9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2115441239485583360","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T13:51:09Z","started_at":"2026-10-28T13:51:09Z","completed_at":"2026-10-28T16:12:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T16:18:07.564171225Z","timing_metadata_checked_at":"2026-10-28T16:18:07.564171225Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T16:18:07.567310715Z","region_metadata_checked_at":"2026-10-28T16:18:07.567310715Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2115375462514429952","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T09:29:46Z","started_at":"2026-10-28T09:29:46Z","completed_at":"2026-10-28T11:57:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T12:03:59.414642155Z","timing_metadata_checked_at":"2026-10-28T12:03:59.414642155Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T12:03:59.417564247Z","region_metadata_checked_at":"2026-10-28T12:03:59.417564247Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a23522ac42f1e31970c5bbb64cf42e84dcff22f7b55aff846b06d3cb461cf18b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2115375462514429952","occurred_at":"2026-10-28T09:29:46Z","failed_at":"unknown","signature_id":"bfb9b703f5241dd736b9902336860998ffe6bdf4ba01f9a3a8b4d81786acbd9e","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:510]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3b09d8d2ba10\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:510]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3b09d8d2ba10\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c349dea01a43efbfe8bada3f00e6006e559aa06c48ac45e43b919b7539b25770","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c349dea01a43efbfe8bada3f00e6006e559aa06c48ac45e43b919b7539b25770"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2115262480291729408","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T02:00:49Z","started_at":"2026-10-28T02:00:49Z","completed_at":"2026-10-28T03:54:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T04:11:14.465266275Z","timing_metadata_checked_at":"2026-10-28T04:11:14.465266275Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T04:11:14.468388566Z","region_metadata_checked_at":"2026-10-28T04:11:14.468388566Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f7a5e5c41af8618158bc30749a6be022d5d04e73c1e634171d8f495dfb12116d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2115262480291729408","occurred_at":"2026-10-28T02:00:49Z","failed_at":"e2e","signature_id":"a130358adaef86ab6bdbcc865713e8b92da4c632acc5d4fbbe39da19d0be4f65","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x8979cd42dc0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-d4wglkkdjhfj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\",\n      \"name\": \"4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T03:01:21.21908488Z\",\n      \"endTime\": \"2026-10-28T03:20:21.275752007Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p0i5l3o3i9z8d7r-np-deletable-95t5c-lvd52: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p0i5l3o3i9z8d7r-np-deletable-95t5c-lvd52: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\n            correlation ID: 48b19522-cae3-4702-8aae-982e9a4042a0\n            client request ID: unavailable\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x8979cd42dc0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-d4wglkkdjhfj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\",\n      \"name\": \"4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T03:01:21.21908488Z\",\n      \"endTime\": \"2026-10-28T03:20:21.275752007Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p0i5l3o3i9z8d7r-np-deletable-95t5c-lvd52: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p0i5l3o3i9z8d7r-np-deletable-95t5c-lvd52: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\n            correlation ID: 48b19522-cae3-4702-8aae-982e9a4042a0\n            client request ID: unavailable","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-d4wglkkdjhfj\" to finish creating: GET https://management"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}