{"meta":{"date":"2026-10-11","timezone":"UTC","generated_at":"2026-10-30T13:35:27Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":3,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-11T23:00:04Z","started_at":"2026-10-11T23:00:04Z","completed_at":"2026-10-12T00:45:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T00:58:30.478354889Z","timing_metadata_checked_at":"2026-10-12T00:58:30.478354889Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T00:58:30.483428574Z","region_metadata_checked_at":"2026-10-12T00:58:30.483428574Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"36774bf613da331c47957ed4a013004a9780cdf2def3be9d058d7e4aa4f96bcb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"eb1c33da48c2dd7e1a17df691460f82388ce7033187824a4224223b7cbcc8fe1","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:137]: failed to verify basic cluster health for idms-e2e-hcp-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0x19c6c82ad0e0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x19c6c78ffea0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x19c6c78ffe80\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x19c6c7ea8960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x19c6c8695740\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x19c6c8695720\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x19c6c7b65c20\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:137]: failed to verify basic cluster health for idms-e2e-hcp-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0x19c6c82ad0e0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x19c6c78ffea0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x19c6c78ffe80\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x19c6c7ea8960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x19c6c8695740\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x19c6c8695720\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x19c6c7b65c20\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized"}},{"row_id":"4f0ac55f5eef4de629970d6f2c5f86c6b0b7b040cf64e0e25ab50c6936ad94d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"85e4ca2686ea9b5410efbbf71ef6da967a97fc5a0f3281b695c388c6119013b0","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2478de214db0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9589trzrcm22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9589trzrcm22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n            correlation ID: 1985dc52-0e75-413c-9ede-87cb3e6ad675\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2478de214db0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9589trzrcm22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9589trzrcm22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n            correlation ID: 1985dc52-0e75-413c-9ede-87cb3e6ad675\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5ffb6d180876e3414b5bd73485bf79991420fef7da5484a0ab20aed8647ae9b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"57f4b8a29534a1b97f7efd40c16cb7d3faa7d37950a5b8bae5e2ef89286cdd95","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2482f98dd8f0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kgfhn26xlnc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kgfhn26xlnc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n            correlation ID: 6cddad86-b2cb-402b-b9c5-2d33d805daed\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2482f98dd8f0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kgfhn26xlnc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kgfhn26xlnc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n            correlation ID: 6cddad86-b2cb-402b-b9c5-2d33d805daed\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"62c0dfa14bc9c329ffadf183946ead2294b6040c8d2ca37d8d155714f83505ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"05c29ad480b5bf27166e88948710fee03cc002f61af8efde1454d88889eb8859","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1fc144a8a360\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-48s6s8v68r22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-48s6s8v68r22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n            correlation ID: 308b2783-d950-49dc-bba1-b7f5095895de\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1fc144a8a360\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-48s6s8v68r22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-48s6s8v68r22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n            correlation ID: 308b2783-d950-49dc-bba1-b7f5095895de\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"bc7522491dec07e326d00739fe6b3d2c44c566618715cfdb9f43b4010d7c7725","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"5bfccd42c01d56d5347db2bbce6e7db03d40121dacef58a1a5dc37e3bff7a16f","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x11944cc8930\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x11944de5c00\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x11944de5be0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x11945431e00\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x11944978520\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x11944978500\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x11944c97ae0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x11944cc8930\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x11944de5c00\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x11944de5be0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x11945431e00\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x11944978520\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x11944978500\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x11944c97ae0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized"}},{"row_id":"d9caa32413ed00b32c9351ebd45e879765d815f8e33f796b2cc9925739d980b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"cf7cb1ca2d45d1b8b3bddf2bf8a6201374e2153edb4dc1531f8dc0a98730aadf","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1809749e4ab0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-bt79q5smg9ln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-bt79q5smg9ln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n            correlation ID: 169674f2-a45c-486a-9ed8-ef274e1803f2\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1809749e4ab0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-bt79q5smg9ln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-bt79q5smg9ln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n            correlation ID: 169674f2-a45c-486a-9ed8-ef274e1803f2\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109405118775431168","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-11T22:05:45Z","started_at":"2026-10-11T22:05:45Z","completed_at":"2026-10-12T00:42:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T00:58:30.251434754Z","timing_metadata_checked_at":"2026-10-12T00:58:30.251434754Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T00:58:30.255498242Z","region_metadata_checked_at":"2026-10-12T00:58:30.255498242Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d14675bde38ebe616c7cb9c10b3d5e8956cef554a7c8f4e5682094648004f4f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109405118775431168","occurred_at":"2026-10-11T22:05:45Z","failed_at":"e2e","signature_id":"7581ac8d1a8bb671ccd2233ea3ec2804c211a820c9531c5cee88d22d89400cc4","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x5325624e738\u003e: \n    resource group \"cni-cilium-g76pf6t9h842\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-g76pf6t9h842\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\",\n      \"name\": \"179cc551-21d3-4583-b554-f672e37b3a84\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T22:50:53.504973281Z\",\n      \"endTime\": \"2026-10-11T23:14:57.954117538Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tcruuiers9mtsh6kof4evdrtdk9p3e4 still exists (deletion dispatched at 2026-10-11T22:50:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x5325624e738\u003e: \n    resource group \"cni-cilium-g76pf6t9h842\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-g76pf6t9h842\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\",\n      \"name\": \"179cc551-21d3-4583-b554-f672e37b3a84\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T22:50:53.504973281Z\",\n      \"endTime\": \"2026-10-11T23:14:57.954117538Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tcruuiers9mtsh6kof4evdrtdk9p3e4 still exists (deletion dispatched at 2026-10-11T22:50:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"86bebf5e619b1e97e49719a882eec1e5a32f3850fd6a0660d27a34314e24e439","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["86bebf5e619b1e97e49719a882eec1e5a32f3850fd6a0660d27a34314e24e439"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2109101693110063104","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-11T02:00:03Z","started_at":"2026-10-11T02:00:03Z","completed_at":"2026-10-11T04:19:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-11T04:30:06.66084184Z","timing_metadata_checked_at":"2026-10-11T04:30:06.66084184Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-11T04:30:06.697334727Z","region_metadata_checked_at":"2026-10-11T04:30:06.697334727Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}