{"meta":{"start_date":"2026-10-14","end_date":"2026-10-14","start_at":"2026-10-14T00:00:00Z","end_at":"2026-10-15T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T08:09:07Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":3,"failed_runs":3,"raw_occurrences":10,"matched_occurrences":10,"runs_affected":3},"rows":[{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":5,"after_last_push_count":89,"trend_counts":[5,5,5,4,4,5,5],"trend_range":"2026-10-08..2026-10-14 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":27,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1f3c2b4a7320\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-rdjxgcpqq89z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-rdjxgcpqq89z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n            correlation ID: 7d77a59e-13ec-4222-9731-91c8b7391fdc\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1f3c2b4a7320\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-rdjxgcpqq89z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-rdjxgcpqq89z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n            correlation ID: 7d77a59e-13ec-4222-9731-91c8b7391fdc\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2e90f118a870\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-bc9b2mq7j2fk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-bc9b2mq7j2fk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n            correlation ID: 3b96e01c-3f99-420c-b36e-0060176be840\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2e90f118a870\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-bc9b2mq7j2fk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-bc9b2mq7j2fk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n            correlation ID: 3b96e01c-3f99-420c-b36e-0060176be840\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:198]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x37edba361e60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dfl62znqdqk2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dfl62znqdqk2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n            correlation ID: e614afff-dbc6-4d8e-b107-e0791f585e4e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:198]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x37edba361e60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dfl62znqdqk2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dfl62znqdqk2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n            correlation ID: e614afff-dbc6-4d8e-b107-e0791f585e4e\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"04c4ff9240bfc96328974d8fef05c048d5d9bf97339039ac23a5904e5d2f1192","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"0a7f872dfbda04f97e316c77c7516ead2a6e83d1d2c0a4303791d99981dad0c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"2b1c630a8187adc4f7f91dc9a28db83c70e788e7789784dc7c19973aba7a8c0c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"c4fd6cb7dad1898836db3bfad3553caa39edb467ad0049c93189b74d3c389531","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"fc548b2bf6881c0c3f143ff03e6b6d2641792df41afacd12b79336d46c1411aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f1386f529db6e94f57caba5baeea25a7ed546b620757afe7e7438f6ec564ee32","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-4swfh9bhdjlh\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-08..2026-10-14 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2be35a9dc6c0\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-4swfh9bhdjlh\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-4swfh9bhdjlh\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2be35bcc7190\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2be35a9dc6c0\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-4swfh9bhdjlh\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-4swfh9bhdjlh\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2be35bcc7190\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x130faa8f320\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-bkm7lwv87ztr\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-bkm7lwv87ztr\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x130f942f8e0\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x130faa8f320\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-bkm7lwv87ztr\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-bkm7lwv87ztr\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x130f942f8e0\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"8e0d229e8c273cfdb2618315909e8aec4ed7980e908455d60fe86b2cbd902c92","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2110189368546168832","occurred_at":"2026-10-14T02:00:55Z","signature_id":"3e20acc848c2269da50d9c76e9d94a46ebbb03e0c5238076f79b8df8ef6b3923","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5f71ef76490634700bd21f11fad8eba65488354974b95740342022fcf2257fdb","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-08..2026-10-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x13ac3c718210\u003e: \n    resource group \"rg-nightly-4-22-mf542htcmkph\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-22-8ktwxt\" in resourcegroup=\"rg-nightly-4-22-mf542htcmkph\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cf27134-aa80-4f7f-83e4-ed95ed8adec3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cf27134-aa80-4f7f-83e4-ed95ed8adec3\",\n      \"name\": \"2cf27134-aa80-4f7f-83e4-ed95ed8adec3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:46:31.678539487Z\",\n      \"endTime\": \"2026-10-15T00:10:39.404101975Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tes3uu6rr6ujm331pj4rtffuvukefm7 still exists (deletion dispatched at 2026-10-14T23:46:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cf27134-aa80-4f7f-83e4-ed95ed8adec3\n            correlation ID: 485192e6-34cc-499a-b81c-b88c264c35e8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x13ac3c718210\u003e: \n    resource group \"rg-nightly-4-22-mf542htcmkph\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-22-8ktwxt\" in resourcegroup=\"rg-nightly-4-22-mf542htcmkph\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cf27134-aa80-4f7f-83e4-ed95ed8adec3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cf27134-aa80-4f7f-83e4-ed95ed8adec3\",\n      \"name\": \"2cf27134-aa80-4f7f-83e4-ed95ed8adec3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:46:31.678539487Z\",\n      \"endTime\": \"2026-10-15T00:10:39.404101975Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tes3uu6rr6ujm331pj4rtffuvukefm7 still exists (deletion dispatched at 2026-10-14T23:46:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cf27134-aa80-4f7f-83e4-ed95ed8adec3\n            correlation ID: 485192e6-34cc-499a-b81c-b88c264c35e8\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"5a7293830fa8b7a0efa7bdd9a4ebab7d81eb7d1225302435819c0f93f39c274b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b73a2d41744880d3e5dcca5cb2a2b41591c045d73749d6123e02539cf38c2cbe","failure_pattern":"There is no failure as the matcher passed to Eventually succeeded on its most recent iteration","search_query":"There is no failure as the matcher passed to Eventually succeeded on its most recent iteration","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-08..2026-10-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:444]: Timed out after 90.000s.\ndelegated NS and ACME TXT should resolve consistently on all authoritative nameservers, or the certificate should already be issued\nThere is no failure as the matcher passed to Eventually succeeded on its most recent iteration\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:444]: Timed out after 90.000s.\ndelegated NS and ACME TXT should resolve consistently on all authoritative nameservers, or the certificate should already be issued\nThere is no failure as the matcher passed to Eventually succeeded on its most recent iteration"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2110519049451474944","occurred_at":"2026-10-14T23:52:07Z","signature_id":"f3300b648ee9771e5ef5c9c950ba9a982e44b9a1e600ce64cde2bc827427941d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,0,1,0,1,0,1],"trend_range":"2026-10-08..2026-10-14 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-10T23:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:106]: failed to verify HCP cluster \"cx-rg-hcp-cluster\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x9c519e26018\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x9c519869940\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x9c519869920\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x9c5196cebe0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x9c51a1232c0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x9c51a1232a0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x9c519493900\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:106]: failed to verify HCP cluster \"cx-rg-hcp-cluster\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x9c519e26018\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x9c519869940\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x9c519869920\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x9c5196cebe0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x9c51a1232c0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x9c51a1232a0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x9c519493900\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"7fd4ca2fd062c9e1ce85412b05f7c985a192323cff82d8394f552f4cf5f7a255","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}