{"meta":{"date":"2026-03-09","timezone":"UTC","generated_at":"2026-11-01T00:09:10Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":64,"failed_runs":41,"runs_with_occurrences":41,"runs_with_failure_pattern_matches":32,"runs_with_unmatched_occurrences":9,"runs_without_occurrence_rows":23,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031154631639306240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4375,"pr_state":"closed","pr_sha":"5f83abef619af4278148a16c27c699f25a00c7a4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-09T23:46:14Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4376/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031153756673937408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4376,"pr_state":"closed","pr_sha":"bc6d9934ffd7a7d07fd2eabd9cb7770e44111ae5","final_merged_sha":"0b33790f453ea37c143074eb4e7d7a235407d207","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-09T23:42:47Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031153310936862720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4373,"pr_state":"closed","pr_sha":"489e59bb3017de74d49e246c21dd09c783782a11","final_merged_sha":"489e59bb3017de74d49e246c21dd09c783782a11","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T23:41:00Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031134396899135488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3973,"pr_state":"open","pr_sha":"fee956dd6a80e69211f67537c45bd0a8b4119ea1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T22:25:51Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"435d823f489945469167c47a3afa20aed77f10de2c761318befa1191b2cbd93e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031134396899135488","occurred_at":"2026-03-09T22:25:51Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4374/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031127705570775040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4374,"pr_state":"closed","pr_sha":"d1097e4d55cb9d3f37d85bd1401446e7ff42fefd","final_merged_sha":"d1097e4d55cb9d3f37d85bd1401446e7ff42fefd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T21:59:16Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bb8a6b9856e6242bd98e9c2baa19a5a18c756427268041292a0fb005262ed6be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4374/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031127705570775040","occurred_at":"2026-03-09T21:59:16Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-09T22:52:02.814Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-09T22:53:22.684Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=2ec4a404-3ad3-4659-b870-94473c8bdde3 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=406507b1-68b1-406a-a5a6-a4cd20e8bb09 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj0775040.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj0775040.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j0775040 -n arohcp-prow-dbcs-j0775040 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0775040/providers/Microsoft.Network/dnszones/j0775040.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j0775040-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j0775040-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:877909c7b876e7cb8dd1d0cdb6485b92e9eb0bdd1494df44a308363b7a44b63b \\\\\\n --set global.imageDigest=sha256:877909c7b876e7cb8dd1d0cdb6485b92e9eb0bdd1494df44a308363b7a44b63b \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j0775040-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"700e186e-1b96-4480-8658-aeda0b3bd4f0\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j0775040-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0775040-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j0775040-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4329/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031126699839590400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4329,"pr_state":"closed","pr_sha":"0289e5cba1076ce737ad437633b268498778a383","final_merged_sha":"0289e5cba1076ce737ad437633b268498778a383","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T21:55:16Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4e1b771a80cd3a133885783a0b38406f41ee2113d7cdc2478fd6032c51cf9544","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4329/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031126699839590400","occurred_at":"2026-03-09T21:55:16Z","failed_at":"e2e","signature_id":"46ec8f1163cf69f19139e7935e335148cfe23032126093d3f3f6645b7989170e","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000945460\u003e: \n failed creating external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-w7l2w2\" for cluster=\"ea-cluster\": PUT https://rp.j9590400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w7l2w2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-w7l2w2\\\" for cluster=\\\"ea-cluster\\\": PUT https://rp.j9590400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w7l2w2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0007eb350\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"c61ab021-ab32-4e23-a190-d1d48acb7623\",\n ],\n \"Date\": [\n \"Mon, 09 Mar 2026 23:29:48 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"112\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000944fe0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j9590400.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w7l2w2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\",\n RawPath: \"\",\n OmitHost: false,\n ForceQuery: false,\n RawQuery: \"api-version=2024-06-10-preview\",\n Fragment: \"\",\n RawFragment: \"\",\n },\n Proto: \"HTTP/1.1\",\n ProtoMajor: 1,\n ProtoMinor: 1,\n Header: {\n \"User-Agent\": [\n \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7; linux)\",\n ],\n \"X-Ms-Arm-Resource-System-Data\": [\n \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-09T23:29:39Z\\\"}\",\n ],\n \"X-Ms-Identity-Url\": [\n \"https://dummyhost.identity.azure.net\",\n ],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000945460\u003e: \n failed creating external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-w7l2w2\" for cluster=\"ea-cluster\": PUT https://rp.j9590400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w7l2w2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-w7l2w2\\\" for cluster=\\\"ea-cluster\\\": PUT https://rp.j9590400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w7l2w2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0007eb350\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"c61ab021-ab32-4e23-a190-d1d48acb7623\",\n ],\n \"Date\": [\n \"Mon, 09 Mar 2026 23:29:48 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"112\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000944fe0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j9590400.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w7l2w2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\",\n RawPath: \"\",\n OmitHost: false,\n ForceQuery: false,\n RawQuery: \"api-version=2024-06-10-preview\",\n Fragment: \"\",\n RawFragment: \"\",\n },\n Proto: \"HTTP/1.1\",\n ProtoMajor: 1,\n ProtoMinor: 1,\n Header: {\n \"User-Agent\": [\n \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7; linux)\",\n ],\n \"X-Ms-Arm-Resource-System-Data\": [\n \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-09T23:29:39Z\\\"}\",\n ],\n \"X-Ms-Identity-Url\": [\n \"https://dummyhost.identity.azure.net\",\n ],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031123053462687744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4272,"pr_state":"open","pr_sha":"66a51589dab1c535676aba03930d89302e1c5b99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T21:40:46Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"846f6f4bf2e4e5b8760d4d36ecb8c5df3f00b6db18e01946813444a5ef02bb44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031123053462687744","occurred_at":"2026-03-09T21:40:46Z","failed_at":"provision","signature_id":"d543605463e7e059f9395c0bf0aca774207b75a2bc744da89b1ddd7e73d603a5","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/management/deploy-velero","test_suite":"step graph","failure_text":"time=2026-03-09T22:32:44.487Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero description=\"Step deploy-velero\\n Kind: Helm\\n\"\ntime=2026-03-09T22:32:45.064Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero values=\"map[azureCloud:AzurePublicCloud azurePlugin:map[digest:sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e repository:oadp/oadp-velero-plugin-for-microsoft-azure-rhel9] bucket:backups clientId:4e5fb508-c971-49fd-b15a-d88ea32903e7 configuration:map[backupPrefix:velero deployNodeAgent:true features:EnableCSI uploaderType:kopia] hypershiftPlugin:map[digest:sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e repository:oadp/oadp-hypershift-velero-plugin-rhel9] imageRegistry:arohcpsvcdev.azurecr.io installJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f installer:map[activeDeadlineSeconds:1800 backoffLimit:3] nodeSelector:map[aro-hcp.azure.com/role:infra] pullBinding:map[workloadIdentityClientId:88657b68-2517-4b35-937a-9968ac0bf3fc workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] resourceGroup:hcp-underlay-prow-j2687744-mgmt-1 storageAccount:oadpprowj26877441 subscriptionId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]] veleroServer:map[digest:sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b repository:oadp/oadp-velero-rhel9]]\"\ntime=2026-03-09T22:32:45.064Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero\ntime=2026-03-09T22:32:45.354Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero namespace=velero\ntime=2026-03-09T22:32:45.663Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero dryRun=false\ntime=2026-03-09T22:32:45.663Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero\ntime=2026-03-09T22:32:45.775Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero\ntime=2026-03-09T22:33:28.396Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero err=\"failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/velero/jobs/velero-install\\\\\\\": Post \\\\\\\"https://guard.69af471ce4e3850001c60012.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": EOF\\\") has prevented the request from succeeding (delete jobs.batch velero-install)\"\ntime=2026-03-09T22:33:28.396Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero\ntime=2026-03-09T22:33:28.396Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero\ntime=2026-03-09T22:33:28.548Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero err=\"failed to get status for release velero: query: failed to query with labels: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/velero/secrets?labelSelector=name%3Dvelero%2Cowner%3Dhelm\\\\\\\": Post \\\\\\\"https://guard.69af471ce4e3850001c60012.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69af471ce4e3850001c60012.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (get secrets)\"\ntime=2026-03-09T22:33:28.548Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero\ntime=2026-03-09T22:33:28.548Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=deploy-velero err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/velero/jobs/velero-install\\\\\\\": Post \\\\\\\"https://guard.69af471ce4e3850001c60012.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": EOF\\\") has prevented the request from succeeding (delete jobs.batch velero-install)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ddbacc994d151f11308f32fdcda88c32ecdf070f85319350b3ce1e0b7831572","failure_pattern":"failed post-install: an error on the server (\"Internal Server Error: \\\\\"/apis/batch/v1/namespaces/velero/jobs/velero-install\\\\\": Post \\\\\"\u003curl\u003e\": EOF\") has prevented the request from succeeding (delete jobs.batch velero-install)","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/velero/jobs/velero-install\\\\\\\": Post \\\\\\\"https://guard.69af471ce4e3850001c60012.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": EOF\\\") has prevented the request from succeeding (delete jobs.batch velero-install)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3ddbacc994d151f11308f32fdcda88c32ecdf070f85319350b3ce1e0b7831572"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4372/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031115189415317504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4372,"pr_state":"closed","pr_sha":"e680d6c0bf7cbb898516f09eff2fab7a910876d4","final_merged_sha":"506b932c6f494bfd84965a40bd51ed4a28647d20","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T21:09:27Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c8ff869831f148345b3282129db7b1e1becf8a17d19595854a5b0047c35cd6a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4372/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031115189415317504","occurred_at":"2026-03-09T21:09:27Z","failed_at":"provision","signature_id":"eae211fc859cfe3523f34c4755b92a87c74b299384c34085b5ad28952b21df2c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-crds","test_suite":"step graph","failure_text":"time=2026-03-09T22:03:06.636Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds description=\"Step deploy-mce-crds\\n Kind: Helm\\n\"\ntime=2026-03-09T22:03:07.168Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds values=map[]\ntime=2026-03-09T22:03:07.168Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-09T22:03:07.658Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds namespace=multicluster-engine\ntime=2026-03-09T22:03:08.145Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds dryRun=false\ntime=2026-03-09T22:03:08.145Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-09T22:03:08.459Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-09T22:03:11.241Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds err=Unauthorized\ntime=2026-03-09T22:03:11.241Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-09T22:03:11.241Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-09T22:03:11.469Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds release=mce-crds namespace=multicluster-engine status=failed description=\"Release \\\"mce-crds\\\" failed: Unauthorized\" values=map[]\ntime=2026-03-09T22:03:11.469Z level=INFO msg=\"No resource info available in release, skipping resource diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds infoNil=false resourceCount=0\ntime=2026-03-09T22:03:11.470Z level=INFO msg=\"Kusto deep link for namespace-level troubleshooting\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds kustoLinkNamespace=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/3yPzWqFMBSE9z7FISsFlRihtIJv0GVXbkqMg00xPyRHpODDF7uwd3V3w8cMM2OCZ2090ntYc3HS8YUEYuuQWbtIM/gAPJWLZly4FEqql0b2jXz7UGqQ/SBfJ1FR29ITT9dNoqruAq8dctQGn5eicSTh9o2t2fbMSA38aj1EcVJM4RuG/yfVtIW1phiWv2xxUkgLEs0/NyOdTf1wQmfzGwAA//91DCRa6AAAAA==\"\ntime=2026-03-09T22:03:11.470Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-09T22:03:11.470Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds err=\"error running Helm release deployment Step, failed to deploy helm release: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3f5a2d82a327ba9cb1fbefb6d8dd1ec1c24c537d00d9bb445cff295c67dc36e2","failure_pattern":"Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3f5a2d82a327ba9cb1fbefb6d8dd1ec1c24c537d00d9bb445cff295c67dc36e2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4329/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031104100795420672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4329,"pr_state":"closed","pr_sha":"0289e5cba1076ce737ad437633b268498778a383","final_merged_sha":"0289e5cba1076ce737ad437633b268498778a383","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T20:25:28Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"83cd7cb76b2ae026fa5b31e20806a0618e821be6cbcdc8afa7bfd364b033c2fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4329/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031104100795420672","occurred_at":"2026-03-09T20:25:28Z","failed_at":"provision","signature_id":"14e1139051b92a0259280ab4bba3ccc679edd3541b36e2204a8d4ec84f155427","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"step graph","failure_text":"time=2026-03-09T21:17:09.631Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-09T21:17:10.317Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:e7827e7a-287a-482f-963b-217e1f5fe28e]] prometheusSpec:map[externalLabels:map[cluster:prow-j5420672-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j5420672-svc-5m0n.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-d185178561f8471d8085adb06d8e0e9f/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-09T21:17:10.317Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T21:17:10.677Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-03-09T21:17:11.040Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-03-09T21:17:11.040Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T21:17:11.148Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T21:17:32.089Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed pre-install: warning: Hook pre-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml failed: rolebindings.rbac.authorization.k8s.io \\\"prometheus-admission\\\" is forbidden: user \\\"c209f8df-52ae-48fb-98ea-380f58b04652\\\" (groups=[\\\"system:authenticated\\\"]) is attempting to grant RBAC permissions not currently held:\\n{APIGroups:[\\\"\\\"], Resources:[\\\"secrets\\\"], Verbs:[\\\"get\\\" \\\"create\\\"]}\"\ntime=2026-03-09T21:17:32.089Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T21:17:32.089Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T21:17:32.242Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed to get status for release arohcp-monitor: query: failed to query with labels: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/prometheus/secrets?labelSelector=name%3Darohcp-monitor%2Cowner%3Dhelm\\\\\\\": Post \\\\\\\"https://guard.69af35a704d2a3000103b7b6.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69af35a704d2a3000103b7b6.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (get secrets)\"\ntime=2026-03-09T21:17:32.243Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T21:17:32.243Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed pre-install: warning: Hook pre-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml failed: rolebindings.rbac.authorization.k8s.io \\\"prometheus-admission\\\" is forbidden: user \\\"c209f8df-52ae-48fb-98ea-380f58b04652\\\" (groups=[\\\"system:authenticated\\\"]) is attempting to grant RBAC permissions not currently held:\\n{APIGroups:[\\\"\\\"], Resources:[\\\"secrets\\\"], Verbs:[\\\"get\\\" \\\"create\\\"]}\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"09e29cda6057345b49a9cd3b0f25cfb94b4d06db13e94a77727bdfde3f8a1ffd","failure_pattern":"failed pre-install: warning: Hook pre-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml failed: rolebindings.rbac.authorization.k8s.io \"prometheus-admission\" is forbidden: user","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed pre-install: warning: Hook pre-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml failed: rolebindings.rbac.authorization.k8s.io \\\"prometheus-admission\\\" is forbidden: user \\\"c209f8df-52ae-48fb-98ea-380f58b04652\\\" (groups=[\\\"system:authenticated\\\"]) is attempting to grant RBAC permissions not currently held:\\n{APIGroups:[\\\"\\\"], Resources:[\\\"secrets\\\"], Verbs:[\\\"get\\\" \\\"create\\\"]}"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["09e29cda6057345b49a9cd3b0f25cfb94b4d06db13e94a77727bdfde3f8a1ffd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031102882001981440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4373,"pr_state":"closed","pr_sha":"489e59bb3017de74d49e246c21dd09c783782a11","final_merged_sha":"489e59bb3017de74d49e246c21dd09c783782a11","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T20:20:17Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9cd8145b5a753dc15e50bcbb25b698298ae280e16f2c0ad043f094d2832fcd2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031102882001981440","occurred_at":"2026-03-09T20:20:17Z","failed_at":"provision","signature_id":"189237e63a8aa17cf7280c69b65c23a633ffc88ff5d1885f4b3849675804c3d6","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/management/deploy","test_suite":"step graph","failure_text":"time=2026-03-09T21:17:10.786Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-09T21:17:11.429Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy values=\"map[azure:map[clientId:7f382cdb-59e9-4015-8c3d-d83c2ecb660a cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] broker:map[host:arohcp-prow-maestro-j1981440.westus3-1.ts.eventgrid.azure.net port:8883] consumerName:hcp-underlay-j1981440-mgmt-1 credsKeyVault:map[name:ah-prow-mg-j1981440-1 secret:hcp-underlay-j1981440-mgmt-1] deployment:map[zoneCount:4] glog_v:2 image:map[digest:sha256:391b002a10ffd994fa4f47e4d1096bac91e512769c0073df782f3cd8d8fe43e7 registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] installAppliedManifestWorkCRD:false replicas:3 serviceAccountName:maestro sideCar:map[image:map[digest:sha256:2dccfca276d1e7611442dc997ac15055f76c4caff92b668b40907cdd68fffa14 registry:mcr.microsoft.com repository:azurelinux/base/nginx]]]\"\ntime=2026-03-09T21:17:11.430Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-09T21:17:11.665Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy namespace=maestro\ntime=2026-03-09T21:17:11.950Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy dryRun=false\ntime=2026-03-09T21:17:11.950Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-09T21:17:12.064Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-09T21:17:17.132Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy err=\"create: failed to create: Unauthorized\"\ntime=2026-03-09T21:17:17.132Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-09T21:17:17.132Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-09T21:17:18.078Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy err=\"failed to get status for release maestro-agent: query: failed to query with labels: Unauthorized\"\ntime=2026-03-09T21:17:18.078Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-09T21:17:18.079Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: create: failed to create: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82051b3fe3ade36d94e25e5df07830f73170ae636dad9efb1d1d600505f66d35","failure_pattern":"create: failed to create: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: create: failed to create: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82051b3fe3ade36d94e25e5df07830f73170ae636dad9efb1d1d600505f66d35"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031102968853434368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4371,"pr_state":"closed","pr_sha":"11ecb878ec9a3501bb7ba58e755d71f6abf8de21","final_merged_sha":"e0b528c4354bf750dd12c0c682e19c640905138e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T20:20:07Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ca120218270fb11e444d0b687d7a4794ea4e16bff581462a786139ab83331fc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031102968853434368","occurred_at":"2026-03-09T20:20:07Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031099560876576768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4373,"pr_state":"closed","pr_sha":"cdd4611b825097f6ede95a3eed8db5e9a5b5f9ec","final_merged_sha":"489e59bb3017de74d49e246c21dd09c783782a11","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T20:05:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a9ddee7cfc90ef3635bd63f97ae3532ddfbf207cd4a6b4df3b3f413f56c30bbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031099560876576768","occurred_at":"2026-03-09T20:05:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4372/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031095111080742912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4372,"pr_state":"closed","pr_sha":"466bfe50a7bad28b1c3b04d07f6dba16621c4846","final_merged_sha":"506b932c6f494bfd84965a40bd51ed4a28647d20","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T19:49:44Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a4530017a85e550ad003d80164510fdaa14c9c0d81845dc00fd3639d631fa033","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4372/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031095111080742912","occurred_at":"2026-03-09T19:49:44Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-09T20:23:54.870Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-09T20:23:57.463Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-09T20:23:59.226Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=9d1c3c7063ba86ccef351ee518270ecb2de5671fe2fe32ffbe110f90b1faa36a\ntime=2026-03-09T20:28:30.393Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0742912-svc/providers/Microsoft.Resources/deployments/9d1c3c7063ba86ccef351ee518270ecb2de5671fe2fe32ffbe110f90b1faa36a/operationStatuses/08584285190472100992\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0742912-svc/providers/Microsoft.Resources/deployments/cluster-kgxhfhv2tbiuw\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0742912-svc/providers/Microsoft.ContainerService/managedClusters/prow-j0742912-svc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563","failure_pattern":"ERROR CODE: DeploymentFailed; detail code GetMSICredentialError; detail message Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4366/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031092026295980032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4366,"pr_state":"closed","pr_sha":"c7c9d81a876290f11802e68dbceb40c40dfd998e","final_merged_sha":"c7c9d81a876290f11802e68dbceb40c40dfd998e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T19:34:07Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"084455b6cbdfad0ca420655300738841237c905e47416e86463853d2f52f987a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4366/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031092026295980032","occurred_at":"2026-03-09T19:34:07Z","failed_at":"provision","signature_id":"14e1139051b92a0259280ab4bba3ccc679edd3541b36e2204a8d4ec84f155427","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"step graph","failure_text":"time=2026-03-09T20:28:06.625Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-09T20:28:07.464Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:ebccaf49-586b-4097-9337-4377e4b3d590]] prometheusSpec:map[externalLabels:map[cluster:prow-j5980032-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j5980032-svc-mz7g.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-d2bd77058e8c421ca34156ba89ea599d/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-09T20:28:07.464Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:28:07.893Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-03-09T20:28:08.224Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-03-09T20:28:08.224Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:28:08.395Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:28:11.917Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed to install CRD crds/crd-prometheuses.yaml: Unauthorized\"\ntime=2026-03-09T20:28:11.917Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:28:11.917Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:28:12.034Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed to get status for release arohcp-monitor: release: not found\"\ntime=2026-03-09T20:28:12.034Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:28:12.035Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed to install CRD crds/crd-prometheuses.yaml: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d93054dca69800e303be30b5868394f35d4801ee3e1c11145f7e3f4792f51f47","failure_pattern":"failed to install CRD crds/crd-prometheuses.yaml: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to install CRD crds/crd-prometheuses.yaml: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d93054dca69800e303be30b5868394f35d4801ee3e1c11145f7e3f4792f51f47"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031093729699303424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4313,"pr_state":"closed","pr_sha":"40bc11d7af0e82eb8a36a7b77d76c911d23c2191","final_merged_sha":"40bc11d7af0e82eb8a36a7b77d76c911d23c2191","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T19:27:43Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b327d26d764ff7996d10646632afc571efc44fe5610d76232892fc363d28690b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031093729699303424","occurred_at":"2026-03-09T19:27:43Z","failed_at":"provision","signature_id":"14e1139051b92a0259280ab4bba3ccc679edd3541b36e2204a8d4ec84f155427","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"step graph","failure_text":"time=2026-03-09T20:30:06.402Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-09T20:30:07.054Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:18377044-6808-4a0d-8226-3c720414e5bf]] prometheusSpec:map[externalLabels:map[cluster:prow-j9303424-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j9303424-svc-gdad.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-66f1ce5fcdd547c583736426bc362eb0/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-09T20:30:07.054Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:30:07.387Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-03-09T20:30:07.806Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-03-09T20:30:07.806Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:30:07.984Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:32:29.208Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed post-install: warning: Hook post-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml failed: Unauthorized\"\ntime=2026-03-09T20:32:29.208Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:32:29.208Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:32:30.035Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed to get status for release arohcp-monitor: query: failed to query with labels: Unauthorized\"\ntime=2026-03-09T20:32:30.035Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:32:30.035Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: warning: Hook post-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml failed: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c1fe40642f72dcd925961621a8eff211439594539df7301b9a690e69fc16d0fd","failure_pattern":"failed post-install: warning: Hook post-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml failed: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: warning: Hook post-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml failed: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c1fe40642f72dcd925961621a8eff211439594539df7301b9a690e69fc16d0fd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031087181639979008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4363,"pr_state":"closed","pr_sha":"523a271a3a332b29ad83ffe992284dea47e7a593","final_merged_sha":"523a271a3a332b29ad83ffe992284dea47e7a593","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T19:10:49Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fb899aaacbadc08a56e6569a3480b98e4dcab51f8238f23e45c1f194e860484f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031087181639979008","occurred_at":"2026-03-09T19:10:49Z","failed_at":"provision","signature_id":"031c54fa35aec9e0c04cce1b936b80aba069b2956505852e58193ab6a029a41c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RouteMonitorOperator/management/deploy","test_suite":"step graph","failure_text":"time=2026-03-09T20:12:11.832Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-09T20:12:12.478Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy values=\"map[imageDigestBlackbox:sha256:b04a9fef4fa086a02fc7fcd8dcdbc4b7b35cc30cdee860fdc6a19dd8b208d63e imageDigestManager:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageDigestOperator:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageRegistryBlackbox:arohcpsvcdev.azurecr.io imageRegistryManager:arohcpsvcdev.azurecr.io imageRegistryOperator:arohcpsvcdev.azurecr.io imageRepositoryBlackbox:prometheus/blackbox-exporter imageRepositoryManager:app-sre/route-monitor-operator imageRepositoryOperator:app-sre/route-monitor-operator]\"\ntime=2026-03-09T20:12:12.478Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-09T20:12:12.748Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy namespace=openshift-route-monitor-operator\ntime=2026-03-09T20:12:13.031Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy dryRun=false\ntime=2026-03-09T20:12:13.031Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-09T20:12:13.138Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-09T20:12:23.616Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=\"Unauthorized\\nUnauthorized\\nUnauthorized\"\ntime=2026-03-09T20:12:23.616Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-09T20:12:23.616Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-09T20:12:26.076Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy release=route-monitor-operator namespace=openshift-route-monitor-operator status=failed description=\"Release \\\"route-monitor-operator\\\" failed: Unauthorized\\nUnauthorized\\nUnauthorized\" values=\"map[imageDigestBlackbox:sha256:b04a9fef4fa086a02fc7fcd8dcdbc4b7b35cc30cdee860fdc6a19dd8b208d63e imageDigestManager:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageDigestOperator:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageRegistryBlackbox:arohcpsvcdev.azurecr.io imageRegistryManager:arohcpsvcdev.azurecr.io imageRegistryOperator:arohcpsvcdev.azurecr.io imageRepositoryBlackbox:prometheus/blackbox-exporter imageRepositoryManager:app-sre/route-monitor-operator imageRepositoryOperator:app-sre/route-monitor-operator]\"\ntime=2026-03-09T20:12:26.077Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yTT2/TQBTE7/4UTz4lkl3Zbv44QT0A4oRAiHICcVjvTpzN/nN310mD8uGRCaFRKaitep0d/d57o51EI5JHcL3nCHRFgkUWWaMxcs0GPL6XVixD9NK2GR2lj8zgj4Qt7C8ldIyf5PG3hIgo/aQZh4GNaUapYiE3zsroBku+ZlwNsutgw1quYu5dH3Gy5K6DZ9H5NLvHeiOtkLZ9CaTTku9fAHQNv5Ucrzl3/fHYh/152IcI8xT0W92HCP/ZafyDmy9qvdveuk6VfGL4/vZmUodWL6pmNm+bds93hVJi4kXw0wHxAPks0r8H5CU38x/cV2XNq5vVpi8bsdpA+LZeTKcNpFIFt7N5p2tWP2fAsy84hdJ5ZxDX6EOu6vCUdD8wy1qI35te4/xzCaxYr+OjcMn3V4nqG3iLiPBu6ERIDrRbw4OiNAiRmY4axB1gaSRYxCCP0qqoZnlxmReLL1WxLKtlefk1HdPFBf3HUw2ecXKgjZOWlLTiSloLf9ZkZ+muwOfNvV/Z5ECdd8Pr3aIZdU4czY+DZKRdm5FBCKwdkM4LeGr2Z8cLBP4zAAD//8VOmdJvBAAA\"\ntime=2026-03-09T20:12:26.077Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-09T20:12:26.078Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: Unauthorized\\nUnauthorized\\nUnauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6be070b587ba841b8e5f1a904ffdf84b3282185d51681a672942d8fdf5ad9259","failure_pattern":"Unauthorized Unauthorized Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: Unauthorized\\nUnauthorized\\nUnauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6be070b587ba841b8e5f1a904ffdf84b3282185d51681a672942d8fdf5ad9259"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031084643226226688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4363,"pr_state":"closed","pr_sha":"523a271a3a332b29ad83ffe992284dea47e7a593","final_merged_sha":"523a271a3a332b29ad83ffe992284dea47e7a593","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T19:07:59Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0f7bb091829c972637a54b8f04d19060dff4be5a0602b42617d9a6d914863429","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031084643226226688","occurred_at":"2026-03-09T19:07:59Z","failed_at":"provision","signature_id":"14e1139051b92a0259280ab4bba3ccc679edd3541b36e2204a8d4ec84f155427","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"step graph","failure_text":"time=2026-03-09T20:02:27.496Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-09T20:02:28.124Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:e24f4666-9dad-4bcf-a6a8-810afacceda9]] prometheusSpec:map[externalLabels:map[cluster:prow-j6226688-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j6226688-svc-itz1.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-2883643e837c4ecea61b4287c67a15e9/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-09T20:02:28.124Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T20:02:29.126Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace prometheus: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ecf60ba8bdd57b40e995d27f45ca5a2fb0a8082c735c55d75d0af026c1021bb","failure_pattern":"failed to apply namespace prometheus: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace prometheus: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8ecf60ba8bdd57b40e995d27f45ca5a2fb0a8082c735c55d75d0af026c1021bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031089531410714624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4362,"pr_state":"closed","pr_sha":"a72618f2244da3fc7983c01e47c67c3a5db80ff9","final_merged_sha":"a72618f2244da3fc7983c01e47c67c3a5db80ff9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T19:05:07Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c4151056f96381589d9daf312506389fcb6d19d7f2fea1bea25e945fb966f29e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031089531410714624","occurred_at":"2026-03-09T19:05:07Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031075261474148352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4371,"pr_state":"closed","pr_sha":"11ecb878ec9a3501bb7ba58e755d71f6abf8de21","final_merged_sha":"e0b528c4354bf750dd12c0c682e19c640905138e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T18:29:39Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f7be53d1be823dd9ffc35a2aa672d255cb081c3a56ee6f3a51e782ac230c7635","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031075261474148352","occurred_at":"2026-03-09T18:29:39Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031070671357874176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4361,"pr_state":"closed","pr_sha":"5b9af084c4315b645526849e9e40d9cae69e886f","final_merged_sha":"5b9af084c4315b645526849e9e40d9cae69e886f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T17:55:53Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"cefa66bc1519a963e92f7b8d1b04dc9c1f2873ab1c43fe8b9789682ec153ca81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031070671357874176","occurred_at":"2026-03-09T17:55:53Z","failed_at":"provision","signature_id":"3098b8b454af6744d84b457d6643a8b0250d9756ef7e2e15ca038dda91e8bbed","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-09T19:05:18.347Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-09T19:05:19.460Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:0edffd03-5e66-4526-b4d5-f391fcc114f1 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j7874176] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j7874176.hcpsvc.osadev.cloud] image:map[digest:sha256:c1ae07dbef1925a1b2f02410f442fdc94d2a2ac8c3ff32d64db02d07be4d69fb registry:arohcpsvcdev.azurecr.io repository:ci-op-9fxy9dxs/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-9fxy9dxs/pipeline:pull workloadIdentityClientId:c2d7b981-86f9-402f-b460-b243cd4e9e83 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:09b8b89e-472d-4280-a194-30a949204bef workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-09T19:05:19.460Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-09T19:05:19.691Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-03-09T19:05:19.993Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-03-09T19:05:19.993Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-09T19:05:20.106Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-09T19:10:28.975Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"resource not ready, name: sessiongate, kind: Deployment, status: Unknown\\ncontext deadline exceeded\"\ntime=2026-03-09T19:10:28.975Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-09T19:10:28.975Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-09T19:10:33.464Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: resource not ready, name: sessiongate, kind: Deployment, status: Unknown\\ncontext deadline exceeded\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:0edffd03-5e66-4526-b4d5-f391fcc114f1 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j7874176] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j7874176.hcpsvc.osadev.cloud] image:map[digest:sha256:c1ae07dbef1925a1b2f02410f442fdc94d2a2ac8c3ff32d64db02d07be4d69fb registry:arohcpsvcdev.azurecr.io repository:ci-op-9fxy9dxs/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-9fxy9dxs/pipeline:pull workloadIdentityClientId:c2d7b981-86f9-402f-b460-b243cd4e9e83 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:09b8b89e-472d-4280-a194-30a949204bef workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-09T19:10:33.466Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5RUTW/bMAy9+1cIPjVAZKQdNqwZeujHgAEDhiDrqcMOsszaamXRI6kWKfLjBzfJYjjOx24S/fT46Ecy8SCKgDGSBVZXqjBixOQezjB/AivfXSimLORCOVar0A9Tw78QvEB4j3Bj7CY8+pUopVR6B43HRQ1B0rFKGZgdhtII7FyP4LUFEvfobHsmeCTgCqgFmWfWjsWhdqEkYN5wXVuaRe9vXChcKFtoE73X+fY+IOA6SoXk3ow4DDP0zi7ek3iPr/00/09Qg5CzrPn4nzjIE1Cq/TV8u7+fzTEOUA/CbyML1vN1C9zBowuuTdqBc9Z5lxlCXdkmM2+RILNYt8gN2xx9P5G2GITQ+5VhAxIOP9IeTAGkwYPtCeuT3Aya2+U6bP9pHD09Rzh/Ar04e6IZc/gTgaV1H4K07b5bsKYVSJso1d6klkBmhC+uALr1ZqdhNdvm4PysdV9bi/HU+b31kQVoyM/1WdcmmBI2I54mv78kzzEHCiDAX9tlwslSvVZAoMTVwGLqRuUgrwBBnRVGoA2fpReTi0968kFPLu/PL6eTj9Pzy4d0pLJM7cOcT6YXnx/S0ShZqid0QT27UFy5EIA6KxCD2m6+7srr77pkqRrC9utW6Fg1WKzAp5GMlcdyrGpgNmVLiVQAqXzRKb4Atn8DAAD//6g2/yKoBQAA\"\ntime=2026-03-09T19:10:33.469Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-09T19:10:33.469Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: sessiongate, kind: Deployment, status: Unknown\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ebfe0e1bf5101c708c1ea158d69ec58626a635cd974cb0320cc10702549248f","failure_pattern":"resource not ready, name: sessiongate, kind: Deployment, status: Unknown context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: sessiongate, kind: Deployment, status: Unknown\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7ebfe0e1bf5101c708c1ea158d69ec58626a635cd974cb0320cc10702549248f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031073241807720448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4361,"pr_state":"closed","pr_sha":"5b9af084c4315b645526849e9e40d9cae69e886f","final_merged_sha":"5b9af084c4315b645526849e9e40d9cae69e886f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T17:44:02Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3a2dc2eae885503493bc40bab538fa83f4ce58d412b53526e819c580ad6795e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031073241807720448","occurred_at":"2026-03-09T17:44:02Z","failed_at":"provision","signature_id":"14e1139051b92a0259280ab4bba3ccc679edd3541b36e2204a8d4ec84f155427","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"step graph","failure_text":"time=2026-03-09T18:59:06.251Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-09T18:59:06.981Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:5a237c08-583c-4b4b-8951-44ba1d90f3e8]] prometheusSpec:map[externalLabels:map[cluster:prow-j7720448-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j7720448-svc-kl72.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-78fa24611df44506bfdbdacb3d78fb5f/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-09T18:59:06.981Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T18:59:07.398Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-03-09T18:59:07.671Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-03-09T18:59:07.671Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T18:59:07.825Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T19:01:05.632Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed post-install: Unauthorized\"\ntime=2026-03-09T19:01:05.632Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T19:01:05.632Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T19:01:25.273Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus release=arohcp-monitor namespace=prometheus status=failed description=\"Release \\\"arohcp-monitor\\\" failed: failed post-install: Unauthorized\" values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:5a237c08-583c-4b4b-8951-44ba1d90f3e8]] prometheusSpec:map[externalLabels:map[cluster:prow-j7720448-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j7720448-svc-kl72.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-78fa24611df44506bfdbdacb3d78fb5f/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-09T19:01:25.274Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yTT2/TQBDF7/4UI58Sya7cogIx6gEKJ1QOgEAq4rDeHZxtvDurmXGrSPnwyHFDDKUoIj36+c1v/mhf1qECo1DPFgUuwBk1apoOZ9TcoNX3PrpalH1sCxilDybgLwlvMW4VScbu5Pm3DAAgv6T4w7dXJuUF5CaYMqCyt1IKqvrYSmm3jjA6Vn2DpaxFMeTFSPhiOu/M4P2KzZJoNSJ7NuopDkWJKaAusZfSuOBF7vUd4bLrRZE/UofbKZiWNpWBolficmypRnE32z9qJ60oIRslfsT9xkfnY3tMwwniv/rui6beT8i33h5+iSllJLzF1NE6YNQngzyy38Oyq16Pegv327+2lvqj5n8I+v3ck6/s+6tswHJERXk3xEWyDdwtkRHUBxQ1IUGDeocYYeaM4iDP8rPq7HlZPSurxefTl/X5oq5eXOdzODmBv3sWdXVaV+fX+XyebeCGfISVj+7Cx4g8CTlF2Gd7Guo/05xtIDENf/eDFpDIjebDIAV01BYQUMS0A5LYIUOznizvUOzPAAAA//8qdpMtigQAAA==\"\ntime=2026-03-09T19:01:25.276Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T19:01:25.276Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6","failure_pattern":"failed post-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4205/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031079190215790592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4205,"pr_state":"closed","pr_sha":"e430bc470c3354d54547e70972016250bdf9af38","final_merged_sha":"a36f37aae6ead63c15a10186a904f97e4e0fc50d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T17:35:54Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7e2075fcd93ecf0e6593b9e5d92d9f503a1e66bf5d3b822180faa929a765eb7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4205/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031079190215790592","occurred_at":"2026-03-09T17:35:54Z","failed_at":"provision","signature_id":"14e1139051b92a0259280ab4bba3ccc679edd3541b36e2204a8d4ec84f155427","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"step graph","failure_text":"time=2026-03-09T19:14:40.550Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-09T19:14:41.306Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:90cdf569-1710-4521-9fc6-1f3ff3ea17c5]] prometheusSpec:map[externalLabels:map[cluster:prow-j5790592-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j5790592-svc-pikb.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-8c209c0d1f4c49e4b5c4c592e6250129/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-09T19:14:41.306Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T19:14:41.619Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-03-09T19:14:42.004Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-03-09T19:14:42.004Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T19:14:42.109Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T19:16:11.008Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed pre-install: Unauthorized\"\ntime=2026-03-09T19:16:11.008Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T19:16:11.009Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T19:16:38.898Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus release=arohcp-monitor namespace=prometheus status=failed description=\"Release \\\"arohcp-monitor\\\" failed: failed pre-install: Unauthorized\" values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:90cdf569-1710-4521-9fc6-1f3ff3ea17c5]] prometheusSpec:map[externalLabels:map[cluster:prow-j5790592-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j5790592-svc-pikb.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-8c209c0d1f4c49e4b5c4c592e6250129/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-09T19:16:38.898Z level=INFO msg=\"No resource info available in release, skipping resource diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus infoNil=false resourceCount=0\ntime=2026-03-09T19:16:38.899Z level=INFO msg=\"Kusto deep link for namespace-level troubleshooting\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus kustoLinkNamespace=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/3SOPYqEQBSEc0/x6EhBxXZFUPAGG25ksrRtoS7bP3S/QQY8/OAEziSTFR9V1KedZbVZhG+3xOSgfUUA8WYQWRlPE3gHLKWzYpw4FXVVt0X1VVTdj+x62fRNPYqMypI+dtpeylFk2XVglUH0SuP3TDQMJHxwBrziFkVykA/uD5pfJjn9uyUn7+bnJDnIhRmBpvvFSEWdv7mrqB8BAAD//xo9pYDfAAAA\"\ntime=2026-03-09T19:16:38.899Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T19:16:38.899Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed pre-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9e8faf9dd151e5316337afa8f0d694f70b9e3597eb997a02e9a233307595638","failure_pattern":"failed pre-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed pre-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d9e8faf9dd151e5316337afa8f0d694f70b9e3597eb997a02e9a233307595638"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031062993386606592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3820,"pr_state":"closed","pr_sha":"0f2aca02f4e987cd9296fbd703ab0e730061e968","final_merged_sha":"18929061480414495bf9f44eb8e304a2e2fe45eb","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T17:08:50Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2d1275cc281a338651eca6fbc412d2e7881fe927b0a4251aa19df09c4d7e076e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031062993386606592","occurred_at":"2026-03-09T17:08:50Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-09T17:59:19.656Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-09T17:59:20.363Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:d01706ad-9272-4795-9df0-cd12fe407df6]] prometheusSpec:map[externalLabels:map[cluster:prow-j6606592-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j6606592-mgmt-1-x3ef.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-f37991ed6afb4738b97c5931ddf103be/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j6606592-mgmt-1-x3ef.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-86a5f5e8e8dc4484a806e3a406e53c23/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-09T17:59:20.363Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-09T17:59:20.636Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus namespace=prometheus\ntime=2026-03-09T17:59:20.958Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus dryRun=false\ntime=2026-03-09T17:59:20.958Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-09T17:59:21.111Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-09T18:00:10.302Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed post-install: Unauthorized\"\ntime=2026-03-09T18:00:10.302Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-09T18:00:10.302Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-09T18:00:11.064Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed to get status for release arohcp-monitor: query: failed to query with labels: Unauthorized\"\ntime=2026-03-09T18:00:11.064Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-09T18:00:11.065Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6","failure_pattern":"failed post-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031051883866492928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3973,"pr_state":"open","pr_sha":"ac18f0919c2874cd6e651b49308d9507fc2fb8ae","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T16:57:58Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fd829cbdb1f66b1f64249b766f019795e874e20c1bbdccbdd5ce5c9dab522b82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031051883866492928","occurred_at":"2026-03-09T16:57:58Z","failed_at":"provision","signature_id":"14e1139051b92a0259280ab4bba3ccc679edd3541b36e2204a8d4ec84f155427","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"step graph","failure_text":"time=2026-03-09T17:45:11.053Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-09T17:45:11.665Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:9df25c9b-5dca-4c19-badc-fc546b07c864]] prometheusSpec:map[externalLabels:map[cluster:prow-j6492928-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j6492928-svc-w1w1.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-c0f374669a024c3b8646e4b2f2fdcccc/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-09T17:45:11.665Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T17:45:12.058Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-03-09T17:45:12.443Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-03-09T17:45:12.443Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T17:45:12.553Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T17:46:26.498Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed pre-install: Unauthorized\"\ntime=2026-03-09T17:46:26.498Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T17:46:26.498Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T17:46:49.466Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus release=arohcp-monitor namespace=prometheus status=failed description=\"Release \\\"arohcp-monitor\\\" failed: failed pre-install: Unauthorized\" values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:9df25c9b-5dca-4c19-badc-fc546b07c864]] prometheusSpec:map[externalLabels:map[cluster:prow-j6492928-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j6492928-svc-w1w1.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-c0f374669a024c3b8646e4b2f2fdcccc/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-09T17:46:49.467Z level=INFO msg=\"No resource info available in release, skipping resource diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus infoNil=false resourceCount=0\ntime=2026-03-09T17:46:49.467Z level=INFO msg=\"Kusto deep link for namespace-level troubleshooting\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus kustoLinkNamespace=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/3SOv4qDQBjEe5/iYysFFd278zjBN7gylU1Y10EN2T/sfkECPnwwhUmTbvgxw/y0s6wWi/DvpphstM4IIF4MIivjaQCvgKV0VIwdp0JWsimqr6L6O9W/7fdPW8teZFSW9LHTtLLpRZYdB1YZRK80znuiriPhgzPgGbcoko18cBdofpnkdHVTTt6Nz0mykQsjAg33g5GKOn9zV1E/AgAA//841rbS3wAAAA==\"\ntime=2026-03-09T17:46:49.467Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-09T17:46:49.468Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed pre-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9e8faf9dd151e5316337afa8f0d694f70b9e3597eb997a02e9a233307595638","failure_pattern":"failed pre-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed pre-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d9e8faf9dd151e5316337afa8f0d694f70b9e3597eb997a02e9a233307595638"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4362/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031050880932909056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4362,"pr_state":"closed","pr_sha":"a72618f2244da3fc7983c01e47c67c3a5db80ff9","final_merged_sha":"a72618f2244da3fc7983c01e47c67c3a5db80ff9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T16:52:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3c4bcc66fbe5392b541b7a6df3fa8d451ad3446e3f12737baf4e420a23d77268","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4362/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031050880932909056","occurred_at":"2026-03-09T16:52:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4329/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031046975398350848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4329,"pr_state":"closed","pr_sha":"0289e5cba1076ce737ad437633b268498778a383","final_merged_sha":"0289e5cba1076ce737ad437633b268498778a383","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T16:38:18Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"09f6d431a408f034ddc93991628e5b7b0bd329e415cf9de19251ffb4defdb644","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4329/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031046975398350848","occurred_at":"2026-03-09T16:38:18Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-09T17:29:26.530Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-09T17:29:50.041Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=f26802a5-17ea-403a-9a5d-adc7ad08cd71 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=ef2ddc9d-ccea-42c0-919d-e108e1d67595 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj8350848.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj8350848.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j8350848 -n arohcp-prow-dbcs-j8350848 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8350848/providers/Microsoft.Network/dnszones/j8350848.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j8350848-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j8350848-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:877909c7b876e7cb8dd1d0cdb6485b92e9eb0bdd1494df44a308363b7a44b63b \\\\\\n --set global.imageDigest=sha256:877909c7b876e7cb8dd1d0cdb6485b92e9eb0bdd1494df44a308363b7a44b63b \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j8350848-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"af9d0d79-b9c1-4f77-8027-fc7adf9326e5\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j8350848-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8350848-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j8350848-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unable to continue with install: could not get information about the resource AuthorizationPolicy \\\"allow-metrics\\\" in namespace \\\"clusters-service\\\": Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031047736991682560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4329,"pr_state":"closed","pr_sha":"0289e5cba1076ce737ad437633b268498778a383","final_merged_sha":"0289e5cba1076ce737ad437633b268498778a383","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T16:35:26Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6c2f4a86cdc6be9f2d51534b22b820a211cab216d57a6f355fc19392cb559896","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031047736991682560","occurred_at":"2026-03-09T16:35:26Z","failed_at":"provision","signature_id":"692850ccec3627f6f18c0069e48528a9b9f4e27dd84dcc0a7a087aa7c2d7afa7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce","test_suite":"step graph","failure_text":"time=2026-03-09T17:34:17.359Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce description=\"Step deploy-mce\\n Kind: Helm\\n\"\ntime=2026-03-09T17:34:17.954Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-03-09T17:34:17.954Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-09T17:34:18.012Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce namespace=multicluster-engine\ntime=2026-03-09T17:34:18.121Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce dryRun=false\ntime=2026-03-09T17:34:18.121Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-09T17:34:18.230Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-09T17:35:35.855Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed post-install: Unauthorized\"\ntime=2026-03-09T17:35:35.855Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-09T17:35:35.855Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-09T17:35:39.803Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce release=mce namespace=multicluster-engine status=pending-install description=\"Initial install underway\" values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-03-09T17:35:39.804Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yTQW+bQBCF7/yKEadYgsjGUBxXObRVTlV6aHpK1cOyPMMG2CWzu7aI/OMraidxq1L3kOubt9880LyghSOGNZ4lLF1TKZxwomhxYYoHSPdZ6XJtHStdRXSQvogOLxK20L8U2wv5LM++B0RE4R0kw4URhZ1vnZKttw4cQ1dKIzY9WDjD8Q5FbUwz4QujA+zTQf1qWkwRe1936TLfZr25SgeT9rovqzzLUqGSnNunLu1WPnus86IcEX8hf1S6VLp64wX/ypykQ+/zZjM4O3CySLVSpmZ436wSNN1ys3vi2rQWj1OI0y1n8r/dsttxdPxxN8fJ7w/woj6/uQNvlcSt0coZHidHa6s2kINsEVsnHOIOjpW08TY5k+JI/CCl8frsoU3Bgh/vg8YXYA0HezNetA32tKvBIKc6WCe6ngq4HaDpohQOo3wRJvPkXTxfxvOrb4t8vUzXi9V9OKPLS5r0ZOtldh/OZsGeHozS1ChdXiutwSc9NJpe63fauz8LF+ypZzNOX4NG1JvyYP4/SEStqSLqYK2oRqThEkzFcPLxJaz8GQAA//8NlsxjLQQAAA==\"\ntime=2026-03-09T17:35:39.805Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-09T17:35:39.805Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6","failure_pattern":"failed post-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4370/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031047754649702400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4370,"pr_state":"open","pr_sha":"2a2dae464713d6ab2c6a447312f5886a4363e8cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T16:35:10Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"350a594802be824ac7aea9383fa1db23f6f1cc6ec8a0b6cd55b72847e5f53d58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4370/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031047754649702400","occurred_at":"2026-03-09T16:35:10Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031042982651629568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4369,"pr_state":"open","pr_sha":"ecf4a8cd2207eaef930c24d9ea87a14f55550f3d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T16:22:36Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8f7fdf3ffc3b65c7e09ee019897bec9c0f61422cbf7dc6ad702d88fcb0c5e67d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031042982651629568","occurred_at":"2026-03-09T16:22:36Z","failed_at":"provision","signature_id":"5564627504dc0ffa8b2a1cfb61f4b200a8883ded65fa2c1a4914a1dd7bd7f88c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config","test_suite":"step graph","failure_text":"time=2026-03-09T17:14:49.228Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-09T17:14:50.219Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-09T17:14:50.219Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T17:14:50.275Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config namespace=multicluster-engine\ntime=2026-03-09T17:14:50.385Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config dryRun=false\ntime=2026-03-09T17:14:50.385Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T17:14:51.142Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config numVersions=0\ntime=2026-03-09T17:14:51.246Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"\\\"mce-config\\\" has no deployed releases\"\ntime=2026-03-09T17:14:51.246Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T17:14:51.246Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T17:14:51.350Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed to get status for release mce-config: release: not found\"\ntime=2026-03-09T17:14:51.350Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T17:14:51.350Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"error running Helm release deployment Step, failed to deploy helm release: \\\"mce-config\\\" has no deployed releases\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0c3a88aa6fe8b6737ccd3ad314d955b79bc7770861d894b93b01045dfed64c","failure_pattern":"\"mce-config\" has no deployed releases","search_query":"error running Helm release deployment Step, failed to deploy helm release: \\\"mce-config\\\" has no deployed releases"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7f0c3a88aa6fe8b6737ccd3ad314d955b79bc7770861d894b93b01045dfed64c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031045419772940288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4244,"pr_state":"closed","pr_sha":"e7f1e378ae58602463a0b616f3ea3534bbfd785f","final_merged_sha":"e7f1e378ae58602463a0b616f3ea3534bbfd785f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T16:18:18Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"1acda23d695b12c27b3bfccd3dadfad6a4d709fe7efb170bd1f8f38b3aeeb8de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031045419772940288","occurred_at":"2026-03-09T16:18:18Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4350/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031041875493785600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4350,"pr_state":"closed","pr_sha":"5cda4405b398eeaea0070203bce328ea07e2edf1","final_merged_sha":"c117b0a88ebb40e9ee7f4bec2c1680e806014b69","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T16:13:49Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"09f6fdfa6e3d6e8a73c5cf367444d9c529406f6482e62b96b27c52f9990e790a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4350/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031041875493785600","occurred_at":"2026-03-09T16:13:49Z","failed_at":"provision","signature_id":"031c54fa35aec9e0c04cce1b936b80aba069b2956505852e58193ab6a029a41c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RouteMonitorOperator/management/deploy","test_suite":"step graph","failure_text":"time=2026-03-09T17:12:43.063Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-09T17:12:43.686Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy values=\"map[imageDigestBlackbox:sha256:b04a9fef4fa086a02fc7fcd8dcdbc4b7b35cc30cdee860fdc6a19dd8b208d63e imageDigestManager:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageDigestOperator:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageRegistryBlackbox:arohcpsvcdev.azurecr.io imageRegistryManager:arohcpsvcdev.azurecr.io imageRegistryOperator:arohcpsvcdev.azurecr.io imageRepositoryBlackbox:prometheus/blackbox-exporter imageRepositoryManager:app-sre/route-monitor-operator imageRepositoryOperator:app-sre/route-monitor-operator]\"\ntime=2026-03-09T17:12:43.686Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-09T17:12:43.943Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy namespace=openshift-route-monitor-operator\ntime=2026-03-09T17:12:44.217Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy dryRun=false\ntime=2026-03-09T17:12:44.217Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-09T17:12:44.322Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-09T17:12:53.834Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=Unauthorized\ntime=2026-03-09T17:12:53.834Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-09T17:12:53.834Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-09T17:12:54.562Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=\"failed to get status for release route-monitor-operator: kubernetes cluster unreachable: the server has asked for the client to provide credentials\"\ntime=2026-03-09T17:12:54.562Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-09T17:12:54.562Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3f5a2d82a327ba9cb1fbefb6d8dd1ec1c24c537d00d9bb445cff295c67dc36e2","failure_pattern":"Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3f5a2d82a327ba9cb1fbefb6d8dd1ec1c24c537d00d9bb445cff295c67dc36e2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4362/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031030561639763968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4362,"pr_state":"closed","pr_sha":"a72618f2244da3fc7983c01e47c67c3a5db80ff9","final_merged_sha":"a72618f2244da3fc7983c01e47c67c3a5db80ff9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T15:32:14Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"70873b76f41db42b558ac1f56600a09e4f6e020097774768d856f231e95e3f31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4362/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031030561639763968","occurred_at":"2026-03-09T15:32:14Z","failed_at":"provision","signature_id":"5564627504dc0ffa8b2a1cfb61f4b200a8883ded65fa2c1a4914a1dd7bd7f88c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config","test_suite":"step graph","failure_text":"time=2026-03-09T16:23:31.884Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-09T16:23:32.553Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-09T16:23:32.553Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T16:23:32.610Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config namespace=multicluster-engine\ntime=2026-03-09T16:23:32.721Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config dryRun=false\ntime=2026-03-09T16:23:32.721Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T16:23:32.834Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T16:25:18.277Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69aef03a53dc340001ad4fd8.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69aef03a53dc340001ad4fd8.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)\"\ntime=2026-03-09T16:25:18.277Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T16:25:18.277Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T16:25:18.408Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed to get status for release mce-config: query: failed to query with labels: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/multicluster-engine/secrets?labelSelector=name%3Dmce-config%2Cowner%3Dhelm\\\\\\\": Post \\\\\\\"https://guard.69aef03a53dc340001ad4fd8.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69aef03a53dc340001ad4fd8.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (get secrets)\"\ntime=2026-03-09T16:25:18.408Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T16:25:18.409Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69aef03a53dc340001ad4fd8.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69aef03a53dc340001ad4fd8.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2a1f671a46ca05a4f95515404a4035dd108dc9eb264bd0b098221cb0d38c8032","failure_pattern":"failed post-install: an error on the server (\"Internal Server Error: \\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\": Post \\\\\"\u003curl\u003e\": proxy error from localhost:9443 while dialing","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69aef03a53dc340001ad4fd8.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69aef03a53dc340001ad4fd8.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2a1f671a46ca05a4f95515404a4035dd108dc9eb264bd0b098221cb0d38c8032"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031030569336311808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4362,"pr_state":"closed","pr_sha":"a72618f2244da3fc7983c01e47c67c3a5db80ff9","final_merged_sha":"a72618f2244da3fc7983c01e47c67c3a5db80ff9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T15:29:19Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7c8b61538655e8cd651524b286247d6acf77b0f07b187f03b8cc7982bcb3f737","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031030569336311808","occurred_at":"2026-03-09T15:29:19Z","failed_at":"provision","signature_id":"5564627504dc0ffa8b2a1cfb61f4b200a8883ded65fa2c1a4914a1dd7bd7f88c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config","test_suite":"step graph","failure_text":"time=2026-03-09T16:22:01.448Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-09T16:22:02.113Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-09T16:22:02.113Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T16:22:02.170Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config namespace=multicluster-engine\ntime=2026-03-09T16:22:02.284Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config dryRun=false\ntime=2026-03-09T16:22:02.284Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T16:22:02.397Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T16:23:28.267Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69aef02a3078b40001365376.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\\"crypto/rsa: verification error\\\\\\\" while trying to verify candidate authority certificate \\\\\\\"ca\\\\\\\")\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)\"\ntime=2026-03-09T16:23:28.267Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T16:23:28.267Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T16:23:28.415Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed to get status for release mce-config: query: failed to query with labels: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/multicluster-engine/secrets?labelSelector=name%3Dmce-config%2Cowner%3Dhelm\\\\\\\": Post \\\\\\\"https://guard.69aef02a3078b40001365376.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\\"crypto/rsa: verification error\\\\\\\" while trying to verify candidate authority certificate \\\\\\\"ca\\\\\\\")\\\") has prevented the request from succeeding (get secrets)\"\ntime=2026-03-09T16:23:28.415Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T16:23:28.415Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69aef02a3078b40001365376.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\\"crypto/rsa: verification error\\\\\\\" while trying to verify candidate authority certificate \\\\\\\"ca\\\\\\\")\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"38239b79b5b799e6cbc8dad16b9ce3a3e5236d97a0d80408b6d601146c6bf4fa","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["38239b79b5b799e6cbc8dad16b9ce3a3e5236d97a0d80408b6d601146c6bf4fa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031028729752326144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3838,"pr_state":"closed","pr_sha":"78f5869bafec4e96eba0c8b8a20231650cbc7748","final_merged_sha":"78f5869bafec4e96eba0c8b8a20231650cbc7748","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T15:25:58Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031026752515477504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4157,"pr_state":"open","pr_sha":"361dd4bdc4a0bfb22b6acda39c29680e51adafad","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T15:18:06Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"41f71d2d50b0afa140e78c6f6ac118b2e7b75886bbc44466768ab0968f48dcde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031026752515477504","occurred_at":"2026-03-09T15:18:06Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-09T16:11:09.882Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-09T16:14:13.473Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=b54f15bd-5909-4f0f-a1b1-b22df91e2f9a \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=3b6966a1-b1aa-421e-8a73-5f22939748ba \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj5477504.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj5477504.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j5477504 -n arohcp-prow-dbcs-j5477504 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5477504/providers/Microsoft.Network/dnszones/j5477504.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j5477504-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j5477504-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:877909c7b876e7cb8dd1d0cdb6485b92e9eb0bdd1494df44a308363b7a44b63b \\\\\\n --set global.imageDigest=sha256:877909c7b876e7cb8dd1d0cdb6485b92e9eb0bdd1494df44a308363b7a44b63b \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j5477504-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"5e55a69f-a797-48e4-840e-895adda3ba18\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j5477504-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5477504-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j5477504-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/clusters-service/persistentvolumeclaims/ocm-cs-db\\\\\\\": Post \\\\\\\"https://guard.69aeed4dab496b000129e332.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\\"crypto/rsa: verification error\\\\\\\" while trying to verify candidate authority certificate \\\\\\\"ca\\\\\\\")\\\") has prevented the request from succeeding (get persistentvolumeclaims ocm-cs-db)\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031015529912733696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4244,"pr_state":"closed","pr_sha":"e7f1e378ae58602463a0b616f3ea3534bbfd785f","final_merged_sha":"e7f1e378ae58602463a0b616f3ea3534bbfd785f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T14:33:16Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"76da697d84c46768b75229fb496ff98437025d0729302121d147e661b85d719a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031015529912733696","occurred_at":"2026-03-09T14:33:16Z","failed_at":"provision","signature_id":"5564627504dc0ffa8b2a1cfb61f4b200a8883ded65fa2c1a4914a1dd7bd7f88c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config","test_suite":"step graph","failure_text":"time=2026-03-09T15:22:27.370Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-09T15:22:28.065Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-09T15:22:28.065Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T15:22:28.126Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config namespace=multicluster-engine\ntime=2026-03-09T15:22:28.246Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config dryRun=false\ntime=2026-03-09T15:22:28.246Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T15:22:28.363Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T15:24:21.061Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69aee1feab496b000129e323.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\\"crypto/rsa: verification error\\\\\\\" while trying to verify candidate authority certificate \\\\\\\"ca\\\\\\\")\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)\"\ntime=2026-03-09T15:24:21.061Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T15:24:21.061Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T15:24:25.786Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config release=mce-config namespace=multicluster-engine status=failed description=\"Release \\\"mce-config\\\" failed: failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69aee1feab496b000129e323.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\\"crypto/rsa: verification error\\\\\\\" while trying to verify candidate authority certificate \\\\\\\"ca\\\\\\\")\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)\" values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-09T15:24:25.788Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6SUz27bMAzG734KwqcGiIrU+4PNQw9dt1OxDdh26rCDLLGqGpk0JDlBgDz8oNiOvaxtgu7ggz+RP5GfCGYOI3gM3HqFAS5ByyijrByecfWAKt5Y0mWI3pKZQyd9lTXuJVwh7ZTQSDXIs18ZAEB+49oQ0TuMV1ozXTPdWZPPIXespBOqO35E6NK/s8N0aLwS6fP9f926aPtYgWQs4ZBy3alfJEmDNVK6+BulJLW7XDTsrNoIxRQ9O9fdfjzb8Ao9SVI4EO68rHHNfjkFjA2PvRrHlXTTqB/oV1bhlVLcUkwny32ekMmpSX1iVRzp+V9cMqsvs8MFeZpvg+PcIA2vIeq9HaXxquzl4TFeQOhK6xs9SvtoSVsyz0GP2/cy7kG71ZgxHdEJaJjUv8w/SH3mFZ6gnQ74hI3jTar9v6dqN/ojb5znDnXPIaIWai8/hsp+f8iWbYWeMGL4nFZFyLawvkePEG2NIcq6gQrjGpHgTMuIST7Li0XxVixeicX7nxdvyqIoi3e3+QzOz+HJmNdlcXGbz2bZFh7YEiwt6UtLhH6y4Jhg3GvThXa4ybItNJ7T6VjoHBrWXfBpkDk4NnOoMQRpEpK9Rg/VZtK8xqD+BAAA///eIDIahgUAAA==\"\ntime=2026-03-09T15:24:25.790Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-09T15:24:25.790Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69aee1feab496b000129e323.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\\"crypto/rsa: verification error\\\\\\\" while trying to verify candidate authority certificate \\\\\\\"ca\\\\\\\")\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"38239b79b5b799e6cbc8dad16b9ce3a3e5236d97a0d80408b6d601146c6bf4fa","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["38239b79b5b799e6cbc8dad16b9ce3a3e5236d97a0d80408b6d601146c6bf4fa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031013975721775104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4362,"pr_state":"closed","pr_sha":"a72618f2244da3fc7983c01e47c67c3a5db80ff9","final_merged_sha":"a72618f2244da3fc7983c01e47c67c3a5db80ff9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T14:27:19Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"263790df8f2e7ddd2695dc8269c4d297011022208f478c4cd60ac17836077971","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031013975721775104","occurred_at":"2026-03-09T14:27:19Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-09T15:12:33.138Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-09T15:12:33.779Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:d19a5f17-9803-4590-8210-047e474e6bef]] prometheusSpec:map[externalLabels:map[cluster:prow-j1775104-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j1775104-mgmt-1-h7u6.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-755a9b7fa00f4d08a8ad438130445a22/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j1775104-mgmt-1-h7u6.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-e9ed56d6e7704d93a2bd5047f0a57cec/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-09T15:12:33.780Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-09T15:12:34.081Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus namespace=prometheus\ntime=2026-03-09T15:12:34.497Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus dryRun=false\ntime=2026-03-09T15:12:34.497Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-09T15:12:34.672Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-09T15:12:44.243Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"unable to continue with install: could not get information about the resource PodMonitor \\\"prometheus\\\" in namespace \\\"prometheus\\\": Unauthorized\"\ntime=2026-03-09T15:12:44.243Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-09T15:12:44.243Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-09T15:12:44.362Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed to get status for release arohcp-monitor: release: not found\"\ntime=2026-03-09T15:12:44.362Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-09T15:12:44.362Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: unable to continue with install: could not get information about the resource PodMonitor \\\"prometheus\\\" in namespace \\\"prometheus\\\": Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"89b2f0f3415cf96c185533e94d6aed77e7c8b5333cb8c2e31685d920a7f44df8","failure_pattern":"unable to continue with install: could not get information about the resource PodMonitor \"prometheus\" in namespace \"\u003cnamespace\u003e\": Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: unable to continue with install: could not get information about the resource PodMonitor \\\"prometheus\\\" in namespace \\\"prometheus\\\": Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["89b2f0f3415cf96c185533e94d6aed77e7c8b5333cb8c2e31685d920a7f44df8"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4368/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031005626615730176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4368,"pr_state":"closed","pr_sha":"120f259f5a73b077de47f61a001c153754d7c08a","final_merged_sha":"120f259f5a73b077de47f61a001c153754d7c08a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T13:53:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4329/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031002893292670976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4329,"pr_state":"closed","pr_sha":"0289e5cba1076ce737ad437633b268498778a383","final_merged_sha":"0289e5cba1076ce737ad437633b268498778a383","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T13:43:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4368/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031002483194597376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4368,"pr_state":"closed","pr_sha":"8e77d7a8902e809c1be952409526da031574f103","final_merged_sha":"120f259f5a73b077de47f61a001c153754d7c08a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T13:41:34Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f78a22fe9d2d293ecdf26a6ee30aed61fc607ce4b5da43eebe5fc4975abb57c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4368/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031002483194597376","occurred_at":"2026-03-09T13:41:34Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031002281331134464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4361,"pr_state":"closed","pr_sha":"5b9af084c4315b645526849e9e40d9cae69e886f","final_merged_sha":"5b9af084c4315b645526849e9e40d9cae69e886f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T13:40:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3838/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030997245318402048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3838,"pr_state":"closed","pr_sha":"78f5869bafec4e96eba0c8b8a20231650cbc7748","final_merged_sha":"78f5869bafec4e96eba0c8b8a20231650cbc7748","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T13:20:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4366/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030995421106540544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4366,"pr_state":"closed","pr_sha":"c7c9d81a876290f11802e68dbceb40c40dfd998e","final_merged_sha":"c7c9d81a876290f11802e68dbceb40c40dfd998e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T13:13:37Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030993620256952320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4272,"pr_state":"open","pr_sha":"e2ff225011d3a52d8df6f576c1ef3a4b5cc579a1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-09T13:06:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4367/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030983387069550592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4367,"pr_state":"closed","pr_sha":"331cb48149b349697b0ed58704144fbb2e518f6c","final_merged_sha":"6106740ed6ed55580a3bb1e73edee7fe36d87ce1","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-09T12:25:47Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030982453446840320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4363,"pr_state":"closed","pr_sha":"523a271a3a332b29ad83ffe992284dea47e7a593","final_merged_sha":"523a271a3a332b29ad83ffe992284dea47e7a593","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T12:22:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4366/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030975132758970368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4366,"pr_state":"closed","pr_sha":"c7c9d81a876290f11802e68dbceb40c40dfd998e","final_merged_sha":"c7c9d81a876290f11802e68dbceb40c40dfd998e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-09T11:52:59Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"70ce1b33274e98065fa3695c7b7c66155789a2c9e599a8104ea21738d9389f6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4366/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030975132758970368","occurred_at":"2026-03-09T11:52:59Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-09T12:23:14.260Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-09T12:23:16.741Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-09T12:23:18.093Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=4377641c4adac4a3d8c4ed688b61f4475e5a685ffa0600e2ffc07cfa4a0f2cc9\ntime=2026-03-09T12:24:48.493Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8970368/providers/Microsoft.Resources/deployments/4377641c4adac4a3d8c4ed688b61f4475e5a685ffa0600e2ffc07cfa4a0f2cc9/operationStatuses/08584285478881872548\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8970368/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8970368/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j8970368\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report b937a25f-6d64-43a4-891a-cb61c3c6e1d2:3/9/2026 12:24:38 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4040/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030973087914135552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4040,"pr_state":"open","pr_sha":"9e94925f147f55421a2560c9da722ba52bed0e5f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T11:44:52Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"36c9544a368521d41e68064b0a089849d6deff5bd413eb7a1692d05897329bf7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4040/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030973087914135552","occurred_at":"2026-03-09T11:44:52Z","failed_at":"e2e","signature_id":"aa5b88ce642e0ef79ce1d0b3ea2e48194c43a0a541271970a48f5b27b0cb1504","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a4ab20\u003e: \n failed to start credential request: POST https://rp.j4135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bb4v2d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bb4v2d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j4135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bb4v2d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bb4v2d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00131d920\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"316\"],\n \"X-Envoy-Upstream-Service-Time\": [\"1031\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"237cd894-58fe-4457-8f45-5cffa02fb959\",\n ],\n \"Date\": [\n \"Mon, 09 Mar 2026 12:41:48 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a4aa40\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 112, 97, 116, 99, 104, 45, 116, 97, 103, 115, 45, 98, 98, 52, 118, 50, 100, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 112, 97, 116, 99, 104, 45, 116, 97, 103, 115, 45, 99, 108, 117, 115, 116, 101, 114, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 316,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j4135552.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bb4v2d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredenti...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a4ab20\u003e: \n failed to start credential request: POST https://rp.j4135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bb4v2d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bb4v2d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j4135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bb4v2d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bb4v2d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00131d920\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"316\"],\n \"X-Envoy-Upstream-Service-Time\": [\"1031\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"237cd894-58fe-4457-8f45-5cffa02fb959\",\n ],\n \"Date\": [\n \"Mon, 09 Mar 2026 12:41:48 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a4aa40\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 112, 97, 116, 99, 104, 45, 116, 97, 103, 115, 45, 98, 98, 52, 118, 50, 100, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 112, 97, 116, 99, 104, 45, 116, 97, 103, 115, 45, 99, 108, 117, 115, 116, 101, 114, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 316,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j4135552.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bb4v2d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredenti...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"491f7ba60c419a0faa324d3b03672a02667d75e6ba31bc1048f23be270bd76a2","failure_pattern":"ERROR CODE: Conflict; detail message Cannot request credential while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}},{"row_id":"ad91ad6fded87f752810ff3131dcc2c0523f088a3b7b7c226bf298c8f5c2795e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4040/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030973087914135552","occurred_at":"2026-03-09T11:44:52Z","failed_at":"e2e","signature_id":"77c5dc4d208027c0e4183b4af2500ced2256ab5c2c8eb088c14c87ebe44a29ae","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0014489e0\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xznwhl\": PUT https://rp.j4135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xznwhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xznwhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-xznwhl\\\": PUT https://rp.j4135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xznwhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xznwhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0014489c0\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-xznwhl\\\": PUT https://rp.j4135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xznwhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xznwhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000ef7ad0\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"6d879c31-98e6-4e93-9efe-1acea3373990\",\n ],\n \"Date\": [\n \"Mon, 09 Mar 2026 12:41:50 GMT\",\n ],\n \"Content-Length\": [\"345\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2084\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc001448840\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0014489e0\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xznwhl\": PUT https://rp.j4135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xznwhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xznwhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-xznwhl\\\": PUT https://rp.j4135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xznwhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xznwhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0014489c0\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-xznwhl\\\": PUT https://rp.j4135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xznwhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xznwhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000ef7ad0\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"6d879c31-98e6-4e93-9efe-1acea3373990\",\n ],\n \"Date\": [\n \"Mon, 09 Mar 2026 12:41:50 GMT\",\n ],\n \"Content-Length\": [\"345\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2084\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc001448840\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f2f71076ecda30b6ae0d13512c44eed7037c704b65b8be7109f64fb982271ac","failure_pattern":"ERROR CODE: Conflict; detail message Cannot create resource while parent resource is provisioning; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0f2f71076ecda30b6ae0d13512c44eed7037c704b65b8be7109f64fb982271ac","491f7ba60c419a0faa324d3b03672a02667d75e6ba31bc1048f23be270bd76a2"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4351/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030958721256919040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4351,"pr_state":"closed","pr_sha":"eac14dfe2d1e1037f83e2effb8b3fabd24958599","final_merged_sha":"131806c6037c274a80c20df0a02233247a78c57e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T10:47:47Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2bd7a46443901ce67fb723615a1224419b58dd9efe4b423fae3e537255cf1fc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4351/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030958721256919040","occurred_at":"2026-03-09T10:47:47Z","failed_at":"e2e","signature_id":"8a2a63371f185c51c99f8b2dcc2df94bc645b3386a236808aa05735ef13a899a","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:244]: Unexpected error:\n \u003c*errors.joinError | 0xc000f746f0\u003e: \n cluster listing in non-existent resource group: expected error but none occurred\n node pool listing in RG with no clusters: expected error containing keywords 'parent resource not found', got: GET https://rp.j6919040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-r8rsmg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n node pool listing in non-existent resource group: expected error containing keywords 'resource group not found', got: GET https://rp.j6919040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/non-existent-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*errors.errorString | 0xc0014f2440\u003e{\n s: \"cluster listing in non-existent resource group: expected error but none occurred\",\n },\n \u003c*errors.errorString | 0xc0014f2c00\u003e{\n s: \"node pool listing in RG with no clusters: expected error containing keywords 'parent resource not found', got: GET https://rp.j6919040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-r8rsmg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n },\n \u003c*errors.errorString | 0xc0012b70a0\u003e{\n s: \"node pool listing in non-existent resource group: expected error containing keywords 'resource group not found', got: GET https://rp.j6919040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/non-existent-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:244]: Unexpected error:\n \u003c*errors.joinError | 0xc000f746f0\u003e: \n cluster listing in non-existent resource group: expected error but none occurred\n node pool listing in RG with no clusters: expected error containing keywords 'parent resource not found', got: GET https://rp.j6919040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-r8rsmg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n node pool listing in non-existent resource group: expected error containing keywords 'resource group not found', got: GET https://rp.j6919040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/non-existent-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*errors.errorString | 0xc0014f2440\u003e{\n s: \"cluster listing in non-existent resource group: expected error but none occurred\",\n },\n \u003c*errors.errorString | 0xc0014f2c00\u003e{\n s: \"node pool listing in RG with no clusters: expected error containing keywords 'parent resource not found', got: GET https://rp.j6919040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-r8rsmg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n },\n \u003c*errors.errorString | 0xc0012b70a0\u003e{\n s: \"node pool listing in non-existent resource group: expected error containing keywords 'resource group not found', got: GET https://rp.j6919040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/non-existent-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4317/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030955647209574400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4317,"pr_state":"closed","pr_sha":"3832c3f226ce4016ff6187e06101b490c14f1565","final_merged_sha":"3832c3f226ce4016ff6187e06101b490c14f1565","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T10:35:34Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030954801759522816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4317,"pr_state":"closed","pr_sha":"3832c3f226ce4016ff6187e06101b490c14f1565","final_merged_sha":"3832c3f226ce4016ff6187e06101b490c14f1565","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T10:32:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3838/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030954484997296128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3838,"pr_state":"closed","pr_sha":"78f5869bafec4e96eba0c8b8a20231650cbc7748","final_merged_sha":"78f5869bafec4e96eba0c8b8a20231650cbc7748","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T10:30:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4363,"pr_state":"closed","pr_sha":"f758798f5312646231a1a1b167f59b4e0568ad51","final_merged_sha":"523a271a3a332b29ad83ffe992284dea47e7a593","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T09:44:59Z"},"failed_at":["e2e"],"failed_test_count":20,"occurrences":[{"row_id":"02dbed4712fab0f9c5b6a5b63fa9f6e7fbc512065c00c341bf607d3e8822df5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"435b29dfba0818eb0df8afc61ee130856be41e86337d5b58751c1832b68f7a64","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"pp-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create an HCP cluster [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:38\n STEP: creating a resource group @ 03/09/26 10:10:47.829\n \"ts\"=\"2026-03-09 10:10:47.829823\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n STEP: creating cluster parameters @ 03/09/26 10:10:51.891\n STEP: creating customer resources @ 03/09/26 10:10:51.891\n STEP: creating the HCP cluster @ 03/09/26 10:11:51.116\n \"ts\"=\"2026-03-09 10:11:51.116841\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-hcp-cluster\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n STEP: getting credentials @ 03/09/26 10:23:26.705\n STEP: ensuring the cluster is viable @ 03/09/26 10:24:18.259\n STEP: creating the node pool @ 03/09/26 10:24:18.625\n STEP: verifying nodepool DiskStorageAccountType matches framework default @ 03/09/26 10:32:32.065\n STEP: verifying a simple web app can run @ 03/09/26 10:32:32.16\nI0309 10:32:32.617568 761 serving_app.go:120] \"created resources\" namespace=\"e2e-serving-app-7m5kv\" deployment=\"agnhost-server\" service=\"agnhost\" route=\"agnhost\"\nI0309 10:34:44.085566 761 serving_app.go:198] Get \"https://agnhost-e2e-serving-app-7m5kv.apps.aro.r9g5s1z6g5g4g7b.c4ed.j2548992.hcp.osadev.cloud\": dial tcp 4.242.208.5:443: connect: connection timed outfailed to get response from routeurlhttps://agnhost-e2e-serving-app-7m5kv.apps.aro.r9g5s1z6g5g4g7b.c4ed.j2548992.hcp.osadev.cloud\n Route became available in less than 5 minutes: url=https://agnhost-e2e-serving-app-7m5kv.apps.aro.r9g5s1z6g5g4g7b.c4ed.j2548992.hcp.osadev.cloud elapsed=2m11.623483652s\n got successful response from route: response=HTTP/1.1 200 OK\r\n Content-Length: 31\r\n Content-Type: text/plain; charset=utf-8\r\n Date: Mon, 09 Mar 2026 10:34:44 GMT\r\n Set-Cookie: 7b24a1275d9da4fa66633acf82aa53e5=fc49160d68b8f372ae3a8c2fd6ec39e3; path=/; HttpOnly; Secure; SameSite=None\r\n \r\n agnhost-server-8678c6cb89-xx8jm\n \"ts\"=\"2026-03-09 10:34:44.295103\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:34:44.295274\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-zsbbgv\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n \"ts\"=\"2026-03-09 10:34:45.369263\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-zsbbgv\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:34:46.300877\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-8d7dc92573349c6d0052ecc8bc03042b1bfdfade27aae309024a834a40d01d5c.yaml.gz\"\n \"ts\"=\"2026-03-09 10:34:46.300998\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-8d7dc92573349c6d0052ecc8bc03042b1bfdfade27aae309024a834a40d01d5c.yaml.gz\"\n \"ts\"=\"2026-03-09 10:34:46.301090\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:34:46.302428\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-116\"\n \"ts\"=\"2026-03-09 10:34:46.302462\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n \"ts\"=\"2026-03-09 10:34:47.018090\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-116\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:34:47.540716\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"basic-cluster-5dwqb4\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-09 10:34:48.067906\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-116\" \"deployment\"=\"msi-deployment-116\"\n \"ts\"=\"2026-03-09 10:34:48.560596\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"basic-cluster-5dwqb4\" \"deployment\"=\"customer-infra-basic-hcp-cluster-zsbbgv\"\n \"ts\"=\"2026-03-09 10:34:49.634787\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:34:49.635001\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:34:49.635157\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n \"ts\"=\"2026-03-09 10:43:34.015898\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n \"ts\"=\"2026-03-09 10:43:34.015954\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n \"ts\"=\"2026-03-09 10:45:03.662128\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [2055.843 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 2055.844 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\npp-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create an HCP cluster [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:38\n STEP: creating a resource group @ 03/09/26 10:10:47.829\n \"ts\"=\"2026-03-09 10:10:47.829823\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n STEP: creating cluster parameters @ 03/09/26 10:10:51.891\n STEP: creating customer resources @ 03/09/26 10:10:51.891\n STEP: creating the HCP cluster @ 03/09/26 10:11:51.116\n \"ts\"=\"2026-03-09 10:11:51.116841\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-hcp-cluster\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n STEP: getting credentials @ 03/09/26 10:23:26.705\n STEP: ensuring the cluster is viable @ 03/09/26 10:24:18.259\n STEP: creating the node pool @ 03/09/26 10:24:18.625\n STEP: verifying nodepool DiskStorageAccountType matches framework default @ 03/09/26 10:32:32.065\n STEP: verifying a simple web app can run @ 03/09/26 10:32:32.16\nI0309 10:32:32.617568 761 serving_app.go:120] \"created resources\" namespace=\"e2e-serving-app-7m5kv\" deployment=\"agnhost-server\" service=\"agnhost\" route=\"agnhost\"\nI0309 10:34:44.085566 761 serving_app.go:198] Get \"https://agnhost-e2e-serving-app-7m5kv.apps.aro.r9g5s1z6g5g4g7b.c4ed.j2548992.hcp.osadev.cloud\": dial tcp 4.242.208.5:443: connect: connection timed outfailed to get response from routeurlhttps://agnhost-e2e-serving-app-7m5kv.apps.aro.r9g5s1z6g5g4g7b.c4ed.j2548992.hcp.osadev.cloud\n Route became available in less than 5 minutes: url=https://agnhost-e2e-serving-app-7m5kv.apps.aro.r9g5s1z6g5g4g7b.c4ed.j2548992.hcp.osadev.cloud elapsed=2m11.623483652s\n got successful response from route: response=HTTP/1.1 200 OK\r\n Content-Length: 31\r\n Content-Type: text/plain; charset=utf-8\r\n Date: Mon, 09 Mar 2026 10:34:44 GMT\r\n Set-Cookie: 7b24a1275d9da4fa66633acf82aa53e5=fc49160d68b8f372ae3a8c2fd6ec39e3; path=/; HttpOnly; Secure; SameSite=None\r\n \r\n agnhost-server-8678c6cb89-xx8jm\n \"ts\"=\"2026-03-09 10:34:44.295103\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:34:44.295274\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-zsbbgv\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n \"ts\"=\"2026-03-09 10:34:45.369263\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-zsbbgv\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:34:46.300877\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-8d7dc92573349c6d0052ecc8bc03042b1bfdfade27aae309024a834a40d01d5c.yaml.gz\"\n \"ts\"=\"2026-03-09 10:34:46.300998\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-8d7dc92573349c6d0052ecc8bc03042b1bfdfade27aae309024a834a40d01d5c.yaml.gz\"\n \"ts\"=\"2026-03-09 10:34:46.301090\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:34:46.302428\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-116\"\n \"ts\"=\"2026-03-09 10:34:46.302462\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n \"ts\"=\"2026-03-09 10:34:47.018090\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-116\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:34:47.540716\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"basic-cluster-5dwqb4\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-09 10:34:48.067906\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-116\" \"deployment\"=\"msi-deployment-116\"\n \"ts\"=\"2026-03-09 10:34:48.560596\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"basic-cluster-5dwqb4\" \"deployment\"=\"customer-infra-basic-hcp-cluster-zsbbgv\"\n \"ts\"=\"2026-03-09 10:34:49.634787\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:34:49.635001\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:34:49.635157\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n \"ts\"=\"2026-03-09 10:43:34.015898\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n \"ts\"=\"2026-03-09 10:43:34.015954\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"basic-cluster-5dwqb4\"\n \"ts\"=\"2026-03-09 10:45:03.662128\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [2055.843 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 2055.844 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"103f41fdc40b521f1956187c82e03cbe4e85f86c099df89d0dde4dea150b8498","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"87efa4660fd9e974b7b70f5aebd245c0b87bcbcc1e964e1bf95f3a9f6a5fac01","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:37\n STEP: creating a resource group @ 03/09/26 10:10:47.922\n \"ts\"=\"2026-03-09 10:10:47.922440\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n STEP: creating cluster parameters @ 03/09/26 10:10:51.961\n STEP: creating customer resources @ 03/09/26 10:10:51.961\n STEP: creating the HCP cluster @ 03/09/26 10:11:50.899\n \"ts\"=\"2026-03-09 10:11:50.899550\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n STEP: creating a nodepool @ 03/09/26 10:20:06.162\n STEP: attempting to update immutable platform profile fields @ 03/09/26 10:26:49.991\n \"ts\"=\"2026-03-09 10:26:50.346326\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:26:50.346455\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-bkkmlm\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n \"ts\"=\"2026-03-09 10:26:51.662901\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-bkkmlm\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:26:52.574574\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:52.574691\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:52.574771\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:52.575907\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n \"ts\"=\"2026-03-09 10:26:52.575889\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-231\"\n \"ts\"=\"2026-03-09 10:26:53.192223\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n \"ts\"=\"2026-03-09 10:26:53.724707\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-231\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:26:54.167175\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\" \"deployment\"=\"customer-infra-negative-tests-cluster-bkkmlm\"\n \"ts\"=\"2026-03-09 10:26:54.735792\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-231\" \"deployment\"=\"msi-deployment-231\"\n \"ts\"=\"2026-03-09 10:26:55.688377\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:55.688572\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:26:55.688746\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n \"ts\"=\"2026-03-09 10:38:00.981659\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n \"ts\"=\"2026-03-09 10:38:00.981723\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n \"ts\"=\"2026-03-09 10:39:26.956949\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1719.057 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1719.072 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:37\n STEP: creating a resource group @ 03/09/26 10:10:47.922\n \"ts\"=\"2026-03-09 10:10:47.922440\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n STEP: creating cluster parameters @ 03/09/26 10:10:51.961\n STEP: creating customer resources @ 03/09/26 10:10:51.961\n STEP: creating the HCP cluster @ 03/09/26 10:11:50.899\n \"ts\"=\"2026-03-09 10:11:50.899550\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n STEP: creating a nodepool @ 03/09/26 10:20:06.162\n STEP: attempting to update immutable platform profile fields @ 03/09/26 10:26:49.991\n \"ts\"=\"2026-03-09 10:26:50.346326\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:26:50.346455\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-bkkmlm\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n \"ts\"=\"2026-03-09 10:26:51.662901\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-bkkmlm\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:26:52.574574\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:52.574691\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:52.574771\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:52.575907\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n \"ts\"=\"2026-03-09 10:26:52.575889\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-231\"\n \"ts\"=\"2026-03-09 10:26:53.192223\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n \"ts\"=\"2026-03-09 10:26:53.724707\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-231\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:26:54.167175\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\" \"deployment\"=\"customer-infra-negative-tests-cluster-bkkmlm\"\n \"ts\"=\"2026-03-09 10:26:54.735792\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-231\" \"deployment\"=\"msi-deployment-231\"\n \"ts\"=\"2026-03-09 10:26:55.688377\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:55.688572\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:26:55.688746\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n \"ts\"=\"2026-03-09 10:38:00.981659\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n \"ts\"=\"2026-03-09 10:38:00.981723\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-negative-tests-brcc4f\"\n \"ts\"=\"2026-03-09 10:39:26.956949\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1719.057 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1719.072 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"2757f583e25a31a54bc8f0a40f256fcf24cfba40994f7963b0fff3817f90dea1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"6cfff11c9137f7918bddf1088a3acbf5f03aa3d6292394378b41996c1b9b638b","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create a cluster with an external auth config and get the external auth config [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:47\n STEP: creating a resource group @ 03/09/26 10:10:47.865\n \"ts\"=\"2026-03-09 10:10:47.866051\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 03/09/26 10:10:52.16\n STEP: creating the HCP cluster @ 03/09/26 10:11:50.705\n \"ts\"=\"2026-03-09 10:11:50.705587\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"ea-cluster\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n STEP: getting credentials @ 03/09/26 10:19:35.761\n STEP: ensuring the cluster is viable @ 03/09/26 10:20:07.832\n STEP: creating the node pool @ 03/09/26 10:20:08.187\n STEP: creating an app registration with a client secret @ 03/09/26 10:29:11.946\n \"ts\"=\"2026-03-09 10:29:11.946635\" \"level\"=0 \"msg\"=\"creating app registration\" \"appName\"=\"aro-hcp-e2e-8562199231633121535\"\n STEP: creating an external auth config with a prefix @ 03/09/26 10:29:16.158\n STEP: verifying ExternalAuth is in a Succeeded state @ 03/09/26 10:29:26.78\n STEP: creating a cluster role binding for the entra application @ 03/09/26 10:29:26.868\n STEP: creating a rest config using OIDC authentication @ 03/09/26 10:29:27.121\n STEP: confirming we can list namespaces using entra OIDC token @ 03/09/26 10:29:27.388\n STEP: creating the console OAuth client secret for external auth @ 03/09/26 10:31:51.388\n STEP: verifying cluster operators are available after external auth config creation @ 03/09/26 10:31:51.659\n \"ts\"=\"2026-03-09 10:31:51.825588\" \"level\"=0 \"msg\"=\"ClusterOperators not yet ready.\" \"unavailable\"=[\"console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)\"]\n \"ts\"=\"2026-03-09 10:32:11.826136\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:32:11.826323\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-ea-cluster-25zhkh\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n \"ts\"=\"2026-03-09 10:32:12.817552\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-ea-cluster-25zhkh\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:32:13.592596\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-5cf63771ae62e0f954d80324422e91fa0bb45dd988e9775341d966a9f013200c.yaml.gz\"\n \"ts\"=\"2026-03-09 10:32:13.592727\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-5cf63771ae62e0f954d80324422e91fa0bb45dd988e9775341d966a9f013200c.yaml.gz\"\n \"ts\"=\"2026-03-09 10:32:13.592807\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:32:13.594091\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-248\"\n \"ts\"=\"2026-03-09 10:32:13.594154\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n \"ts\"=\"2026-03-09 10:32:14.215434\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-248\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:32:14.664526\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\" \"deployment\"=\"pooledNonMsiScopedAssignments-ltdphft7fgrwg\"\n \"ts\"=\"2026-03-09 10:32:15.151040\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-248\" \"deployment\"=\"msi-deployment-248\"\n \"ts\"=\"2026-03-09 10:32:15.575662\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\" \"deployment\"=\"customer-infra-ea-cluster-25zhkh\"\n \"ts\"=\"2026-03-09 10:32:16.542374\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:32:16.542602\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:32:16.542822\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n \"ts\"=\"2026-03-09 10:42:41.717272\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n \"ts\"=\"2026-03-09 10:42:41.717338\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n \"ts\"=\"2026-03-09 10:44:08.504987\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [2000.673 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 2000.673 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create a cluster with an external auth config and get the external auth config [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:47\n STEP: creating a resource group @ 03/09/26 10:10:47.865\n \"ts\"=\"2026-03-09 10:10:47.866051\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 03/09/26 10:10:52.16\n STEP: creating the HCP cluster @ 03/09/26 10:11:50.705\n \"ts\"=\"2026-03-09 10:11:50.705587\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"ea-cluster\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n STEP: getting credentials @ 03/09/26 10:19:35.761\n STEP: ensuring the cluster is viable @ 03/09/26 10:20:07.832\n STEP: creating the node pool @ 03/09/26 10:20:08.187\n STEP: creating an app registration with a client secret @ 03/09/26 10:29:11.946\n \"ts\"=\"2026-03-09 10:29:11.946635\" \"level\"=0 \"msg\"=\"creating app registration\" \"appName\"=\"aro-hcp-e2e-8562199231633121535\"\n STEP: creating an external auth config with a prefix @ 03/09/26 10:29:16.158\n STEP: verifying ExternalAuth is in a Succeeded state @ 03/09/26 10:29:26.78\n STEP: creating a cluster role binding for the entra application @ 03/09/26 10:29:26.868\n STEP: creating a rest config using OIDC authentication @ 03/09/26 10:29:27.121\n STEP: confirming we can list namespaces using entra OIDC token @ 03/09/26 10:29:27.388\n STEP: creating the console OAuth client secret for external auth @ 03/09/26 10:31:51.388\n STEP: verifying cluster operators are available after external auth config creation @ 03/09/26 10:31:51.659\n \"ts\"=\"2026-03-09 10:31:51.825588\" \"level\"=0 \"msg\"=\"ClusterOperators not yet ready.\" \"unavailable\"=[\"console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)\"]\n \"ts\"=\"2026-03-09 10:32:11.826136\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:32:11.826323\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-ea-cluster-25zhkh\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n \"ts\"=\"2026-03-09 10:32:12.817552\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-ea-cluster-25zhkh\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:32:13.592596\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-5cf63771ae62e0f954d80324422e91fa0bb45dd988e9775341d966a9f013200c.yaml.gz\"\n \"ts\"=\"2026-03-09 10:32:13.592727\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-5cf63771ae62e0f954d80324422e91fa0bb45dd988e9775341d966a9f013200c.yaml.gz\"\n \"ts\"=\"2026-03-09 10:32:13.592807\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:32:13.594091\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-248\"\n \"ts\"=\"2026-03-09 10:32:13.594154\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n \"ts\"=\"2026-03-09 10:32:14.215434\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-248\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:32:14.664526\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\" \"deployment\"=\"pooledNonMsiScopedAssignments-ltdphft7fgrwg\"\n \"ts\"=\"2026-03-09 10:32:15.151040\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-248\" \"deployment\"=\"msi-deployment-248\"\n \"ts\"=\"2026-03-09 10:32:15.575662\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\" \"deployment\"=\"customer-infra-ea-cluster-25zhkh\"\n \"ts\"=\"2026-03-09 10:32:16.542374\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:32:16.542602\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:32:16.542822\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n \"ts\"=\"2026-03-09 10:42:41.717272\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n \"ts\"=\"2026-03-09 10:42:41.717338\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"external-auth-cluster-4sjmt9\"\n \"ts\"=\"2026-03-09 10:44:08.504987\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [2000.673 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 2000.673 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"353a8000e1a1f3bbd0d67bb683086fae08593e39613549e7b781d997c28db3d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"e9166bd24059293a551fb05406966acab39b60e5d745fa41d97c7aa4070272ab","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"ed resource group still exists @ 03/09/26 10:20:11.592\n \"ts\"=\"2026-03-09 10:20:12.407489\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:20:12.407602\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-swqjnv\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:20:13.020583\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-swqjnv\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:20:13.581140\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-4zdhql\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:20:13.718642\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-4zdhql\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:20:15.009391\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster3-bjgscr\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:20:15.116323\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster3-bjgscr\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:20:15.881871\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-09 10:20:15.881997\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-09 10:20:15.882104\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:20:15.883018\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-137\"\n \"ts\"=\"2026-03-09 10:20:15.883087\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-1\"\n \"ts\"=\"2026-03-09 10:20:15.883235\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-101\"\n \"ts\"=\"2026-03-09 10:20:15.883234\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:20:16.578570\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-137\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:20:16.989420\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-1\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:20:17.533363\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-101\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:20:18.050729\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-4lz8v2\" \"deployment\"=\"pooledNonMsiScopedAssignments-igpwm4vimfr3y\"\n \"ts\"=\"2026-03-09 10:20:18.461745\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-137\" \"deployment\"=\"msi-deployment-137\"\n \"ts\"=\"2026-03-09 10:20:18.943848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-1\" \"deployment\"=\"msi-deployment-1\"\n \"ts\"=\"2026-03-09 10:20:19.413908\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-101\" \"deployment\"=\"msi-deployment-101\"\n \"ts\"=\"2026-03-09 10:20:19.998242\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-4lz8v2\" \"deployment\"=\"customer-infra-basic-hcp-cluster3-bjgscr\"\n \"ts\"=\"2026-03-09 10:20:23.020639\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-4lz8v2\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-09 10:20:24.079779\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-4lz8v2\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-4zdhql\"\n \"ts\"=\"2026-03-09 10:20:25.089378\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-4lz8v2\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-09 10:20:26.112562\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-4lz8v2\" \"deployment\"=\"customer-infra-basic-hcp-cluster-swqjnv\"\n \"ts\"=\"2026-03-09 10:20:27.142024\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:20:27.142351\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:20:27.142586\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:29:02.517551\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:29:02.517609\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:30:58.455642\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1210.618 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1210.619 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\ned resource group still exists @ 03/09/26 10:20:11.592\n \"ts\"=\"2026-03-09 10:20:12.407489\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:20:12.407602\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-swqjnv\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:20:13.020583\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-swqjnv\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:20:13.581140\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-4zdhql\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:20:13.718642\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-4zdhql\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:20:15.009391\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster3-bjgscr\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:20:15.116323\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster3-bjgscr\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:20:15.881871\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-09 10:20:15.881997\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-09 10:20:15.882104\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:20:15.883018\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-137\"\n \"ts\"=\"2026-03-09 10:20:15.883087\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-1\"\n \"ts\"=\"2026-03-09 10:20:15.883235\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-101\"\n \"ts\"=\"2026-03-09 10:20:15.883234\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:20:16.578570\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-137\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:20:16.989420\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-1\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:20:17.533363\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-101\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:20:18.050729\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-4lz8v2\" \"deployment\"=\"pooledNonMsiScopedAssignments-igpwm4vimfr3y\"\n \"ts\"=\"2026-03-09 10:20:18.461745\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-137\" \"deployment\"=\"msi-deployment-137\"\n \"ts\"=\"2026-03-09 10:20:18.943848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-1\" \"deployment\"=\"msi-deployment-1\"\n \"ts\"=\"2026-03-09 10:20:19.413908\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-101\" \"deployment\"=\"msi-deployment-101\"\n \"ts\"=\"2026-03-09 10:20:19.998242\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-4lz8v2\" \"deployment\"=\"customer-infra-basic-hcp-cluster3-bjgscr\"\n \"ts\"=\"2026-03-09 10:20:23.020639\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-4lz8v2\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-09 10:20:24.079779\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-4lz8v2\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-4zdhql\"\n \"ts\"=\"2026-03-09 10:20:25.089378\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-4lz8v2\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-09 10:20:26.112562\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-4lz8v2\" \"deployment\"=\"customer-infra-basic-hcp-cluster-swqjnv\"\n \"ts\"=\"2026-03-09 10:20:27.142024\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:20:27.142351\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:20:27.142586\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:29:02.517551\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:29:02.517609\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"customer-rg-4lz8v2\"\n \"ts\"=\"2026-03-09 10:30:58.455642\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1210.618 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1210.619 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"37cbcc810e8e2fee792ead3eb40f822820cff7bff652a5b691c770288b887987","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"f25e975ce7a4dac3c75f7ac0b4b56cdef85398ac857e667334bed20c5a3687f2","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n STEP: creating a resource group @ 03/09/26 10:10:47.93\n \"ts\"=\"2026-03-09 10:10:47.930589\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n STEP: creating cluster parameters @ 03/09/26 10:10:51.915\n STEP: creating customer resources @ 03/09/26 10:10:51.915\n STEP: creating the HCP cluster @ 03/09/26 10:11:49.704\n \"ts\"=\"2026-03-09 10:11:49.705076\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n STEP: creating nodepool parameters with autoscaling enabled @ 03/09/26 10:19:23.725\n STEP: creating the autoscaling nodepool @ 03/09/26 10:19:23.725\n STEP: verifying the cluster has default autoscaling parameters @ 03/09/26 10:26:16.76\n \"ts\"=\"2026-03-09 10:26:16.864222\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:26:16.864347\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-mlfc64\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n \"ts\"=\"2026-03-09 10:26:17.947876\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-mlfc64\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:26:18.850957\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:18.851078\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:18.851173\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:18.852156\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-289\"\n \"ts\"=\"2026-03-09 10:26:18.852285\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n \"ts\"=\"2026-03-09 10:26:19.507463\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-289\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:26:19.954645\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n \"ts\"=\"2026-03-09 10:26:20.458134\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-289\" \"deployment\"=\"msi-deployment-289\"\n \"ts\"=\"2026-03-09 10:26:20.965146\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\" \"deployment\"=\"customer-infra-np-autoscale-cluster-mlfc64\"\n \"ts\"=\"2026-03-09 10:26:21.938401\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:21.938568\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:26:21.938712\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n \"ts\"=\"2026-03-09 10:36:16.972876\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n \"ts\"=\"2026-03-09 10:36:16.972932\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n \"ts\"=\"2026-03-09 10:38:02.868147\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1634.994 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1634.994 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n STEP: creating a resource group @ 03/09/26 10:10:47.93\n \"ts\"=\"2026-03-09 10:10:47.930589\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n STEP: creating cluster parameters @ 03/09/26 10:10:51.915\n STEP: creating customer resources @ 03/09/26 10:10:51.915\n STEP: creating the HCP cluster @ 03/09/26 10:11:49.704\n \"ts\"=\"2026-03-09 10:11:49.705076\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n STEP: creating nodepool parameters with autoscaling enabled @ 03/09/26 10:19:23.725\n STEP: creating the autoscaling nodepool @ 03/09/26 10:19:23.725\n STEP: verifying the cluster has default autoscaling parameters @ 03/09/26 10:26:16.76\n \"ts\"=\"2026-03-09 10:26:16.864222\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:26:16.864347\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-mlfc64\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n \"ts\"=\"2026-03-09 10:26:17.947876\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-mlfc64\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:26:18.850957\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:18.851078\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:18.851173\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:18.852156\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-289\"\n \"ts\"=\"2026-03-09 10:26:18.852285\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n \"ts\"=\"2026-03-09 10:26:19.507463\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-289\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:26:19.954645\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n \"ts\"=\"2026-03-09 10:26:20.458134\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-289\" \"deployment\"=\"msi-deployment-289\"\n \"ts\"=\"2026-03-09 10:26:20.965146\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\" \"deployment\"=\"customer-infra-np-autoscale-cluster-mlfc64\"\n \"ts\"=\"2026-03-09 10:26:21.938401\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:21.938568\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:26:21.938712\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n \"ts\"=\"2026-03-09 10:36:16.972876\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n \"ts\"=\"2026-03-09 10:36:16.972932\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"np-autoscaling-bhcdxt\"\n \"ts\"=\"2026-03-09 10:38:02.868147\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1634.994 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1634.994 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"5fde94bd61b8e83dc1eb5c685160bf80af24f70045c89102e57ab2b96d1914dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"9fc77db00e1c9f0fd2600ead676748caeabde56c78eca94fde123b982ddbe2ae","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"o/runtimeclasses; list admissionregistration.k8s.io/mutatingwebhookconfigurations; list admissionregistration.k8s.io/validatingwebhookconfigurations; list apiregistration.k8s.io/apiservices; list config.openshift.io/clusterversions; list config.openshift.io/clusteroperators; list config.openshift.io/infrastructures; list monitoring.coreos.com/prometheusrules; list monitoring.coreos.com/servicemonitors; list machine.openshift.io/machines; list machine.openshift.io/machinesets; list operator.openshift.io/ingresscontrollers; list security.openshift.io/securitycontextconstraints; list route.openshift.io/routes; list image.openshift.io/imagestreams; create authentication.k8s.io/tokenreviews; create authorization.k8s.io/selfsubjectaccessreviews; create authorization.k8s.io/subjectaccessreviews) @ 03/09/26 10:20:30.497\n STEP: verifying VerifyListNamespaced(namespace=kube-system, resources=secrets) @ 03/09/26 10:20:38.951\n STEP: verifying VerifyRBACAllowed(get /secrets; create /pods; delete /pods; create /namespaces; create apps/deployments; delete apps/deployments; update apps/deployments) @ 03/09/26 10:20:39.013\n STEP: waiting for the session to expire @ 03/09/26 10:20:39.384\n STEP: verifying the session is expired @ 03/09/26 10:22:24.005\n STEP: trying to access a breakglass session of another user @ 03/09/26 10:22:24.23\n STEP: reaching out to the admin API to create a breakglass session for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-hzp78t/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/sre-hcp-cluster with aro-sre-pso permissions: https://admin.j2548992.hcpsvc.osadev.cloud/admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-hzp78t/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/sre-hcp-cluster/breakglass @ 03/09/26 10:22:24.23\n STEP: waiting for SRE breakglass session to be ready at /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-hzp78t/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-7jk85/kubeconfig @ 03/09/26 10:22:24.568\n STEP: and expecting cluster access to be denied @ 03/09/26 10:22:29.624\n \"ts\"=\"2026-03-09 10:22:30.055993\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:22:30.056109\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-sre-hcp-cluster-c7h97c\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\"\n \"ts\"=\"2026-03-09 10:22:31.233312\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-sre-hcp-cluster-c7h97c\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:22:32.033391\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-c550bd9856d8576e40dce24068d9220b0bd5f3dcb700639debaf3003763a0444.yaml.gz\"\n \"ts\"=\"2026-03-09 10:22:32.033526\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-c550bd9856d8576e40dce24068d9220b0bd5f3dcb700639debaf3003763a0444.yaml.gz\"\n \"ts\"=\"2026-03-09 10:22:32.033623\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:22:32.034571\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-45\"\n \"ts\"=\"2026-03-09 10:22:32.034614\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\"\n \"ts\"=\"2026-03-09 10:22:32.629929\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-45\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:22:33.198337\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\" \"deployment\"=\"pooledNonMsiScopedAssignments-456d7mdgworfy\"\n \"ts\"=\"2026-03-09 10:22:33.667913\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-45\" \"deployment\"=\"msi-deployment-45\"\n \"ts\"=\"2026-03-09 10:22:34.360873\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\" \"deployment\"=\"customer-infra-sre-hcp-cluster-c7h97c\"\n \"ts\"=\"2026-03-09 10:22:35.319011\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:22:35.319243\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:22:35.319464\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\"\n \"ts\"=\"2026-03-09 10:30:59.869409\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\"\n \"ts\"=\"2026-03-09 10:30:59.869473\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\"\n \"ts\"=\"2026-03-09 10:32:28.072323\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1300.279 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1300.279 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\no/runtimeclasses; list admissionregistration.k8s.io/mutatingwebhookconfigurations; list admissionregistration.k8s.io/validatingwebhookconfigurations; list apiregistration.k8s.io/apiservices; list config.openshift.io/clusterversions; list config.openshift.io/clusteroperators; list config.openshift.io/infrastructures; list monitoring.coreos.com/prometheusrules; list monitoring.coreos.com/servicemonitors; list machine.openshift.io/machines; list machine.openshift.io/machinesets; list operator.openshift.io/ingresscontrollers; list security.openshift.io/securitycontextconstraints; list route.openshift.io/routes; list image.openshift.io/imagestreams; create authentication.k8s.io/tokenreviews; create authorization.k8s.io/selfsubjectaccessreviews; create authorization.k8s.io/subjectaccessreviews) @ 03/09/26 10:20:30.497\n STEP: verifying VerifyListNamespaced(namespace=kube-system, resources=secrets) @ 03/09/26 10:20:38.951\n STEP: verifying VerifyRBACAllowed(get /secrets; create /pods; delete /pods; create /namespaces; create apps/deployments; delete apps/deployments; update apps/deployments) @ 03/09/26 10:20:39.013\n STEP: waiting for the session to expire @ 03/09/26 10:20:39.384\n STEP: verifying the session is expired @ 03/09/26 10:22:24.005\n STEP: trying to access a breakglass session of another user @ 03/09/26 10:22:24.23\n STEP: reaching out to the admin API to create a breakglass session for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-hzp78t/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/sre-hcp-cluster with aro-sre-pso permissions: https://admin.j2548992.hcpsvc.osadev.cloud/admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-hzp78t/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/sre-hcp-cluster/breakglass @ 03/09/26 10:22:24.23\n STEP: waiting for SRE breakglass session to be ready at /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-hzp78t/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-7jk85/kubeconfig @ 03/09/26 10:22:24.568\n STEP: and expecting cluster access to be denied @ 03/09/26 10:22:29.624\n \"ts\"=\"2026-03-09 10:22:30.055993\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:22:30.056109\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-sre-hcp-cluster-c7h97c\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\"\n \"ts\"=\"2026-03-09 10:22:31.233312\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-sre-hcp-cluster-c7h97c\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:22:32.033391\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-c550bd9856d8576e40dce24068d9220b0bd5f3dcb700639debaf3003763a0444.yaml.gz\"\n \"ts\"=\"2026-03-09 10:22:32.033526\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-c550bd9856d8576e40dce24068d9220b0bd5f3dcb700639debaf3003763a0444.yaml.gz\"\n \"ts\"=\"2026-03-09 10:22:32.033623\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:22:32.034571\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-45\"\n \"ts\"=\"2026-03-09 10:22:32.034614\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\"\n \"ts\"=\"2026-03-09 10:22:32.629929\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-45\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:22:33.198337\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\" \"deployment\"=\"pooledNonMsiScopedAssignments-456d7mdgworfy\"\n \"ts\"=\"2026-03-09 10:22:33.667913\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-45\" \"deployment\"=\"msi-deployment-45\"\n \"ts\"=\"2026-03-09 10:22:34.360873\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\" \"deployment\"=\"customer-infra-sre-hcp-cluster-c7h97c\"\n \"ts\"=\"2026-03-09 10:22:35.319011\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:22:35.319243\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:22:35.319464\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\"\n \"ts\"=\"2026-03-09 10:30:59.869409\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\"\n \"ts\"=\"2026-03-09 10:30:59.869473\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"admin-api-breakglass-hzp78t\"\n \"ts\"=\"2026-03-09 10:32:28.072323\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1300.279 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1300.279 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"6d57bb6815f0e3b749aa025a9cc6cce960233e24204179591e6c601b1d149b63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"c1366f6d0c4798e7a9a55c5d1009dfe07a8bb8974f70492544fc7972193d2219","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:120\n STEP: creating a resource group @ 03/09/26 10:30:58.86\n \"ts\"=\"2026-03-09 10:30:58.860624\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n STEP: creating cluster parameters @ 03/09/26 10:31:01.856\n STEP: creating customer resources @ 03/09/26 10:31:01.856\n STEP: creating the HCP cluster @ 03/09/26 10:32:00.091\n \"ts\"=\"2026-03-09 10:32:00.092051\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"patch-tags-cluster\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n STEP: getting credentials @ 03/09/26 10:38:03.464\n STEP: ensuring the cluster is viable @ 03/09/26 10:38:54.6\n STEP: sending a PATCH request to set a tag @ 03/09/26 10:38:55.028\n STEP: verifying the tag is present in the update response body @ 03/09/26 10:39:35.857\n STEP: verifying the tag is present on the cluster @ 03/09/26 10:39:35.857\n \"ts\"=\"2026-03-09 10:39:35.967590\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:39:35.967743\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-patch-tags-cluster-ddgqtt\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n \"ts\"=\"2026-03-09 10:39:36.966000\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-patch-tags-cluster-ddgqtt\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:39:38.037822\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n \"ts\"=\"2026-03-09 10:39:38.037962\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n \"ts\"=\"2026-03-09 10:39:38.038046\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:39:38.039324\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-1\"\n \"ts\"=\"2026-03-09 10:39:38.039357\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n \"ts\"=\"2026-03-09 10:39:38.663283\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-1\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:39:39.092513\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-pxqr4n\" \"deployment\"=\"pooledNonMsiScopedAssignments-fklo75htpxzxc\"\n \"ts\"=\"2026-03-09 10:39:39.532248\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-1\" \"deployment\"=\"msi-deployment-1\"\n \"ts\"=\"2026-03-09 10:39:39.990307\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-pxqr4n\" \"deployment\"=\"customer-infra-patch-tags-cluster-ddgqtt\"\n \"ts\"=\"2026-03-09 10:39:40.959434\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:39:40.960089\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:39:40.960324\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n \"ts\"=\"2026-03-09 10:48:35.606888\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n \"ts\"=\"2026-03-09 10:48:35.606954\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n \"ts\"=\"2026-03-09 10:50:01.392331\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [2353.450 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 2353.451 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:120\n STEP: creating a resource group @ 03/09/26 10:30:58.86\n \"ts\"=\"2026-03-09 10:30:58.860624\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n STEP: creating cluster parameters @ 03/09/26 10:31:01.856\n STEP: creating customer resources @ 03/09/26 10:31:01.856\n STEP: creating the HCP cluster @ 03/09/26 10:32:00.091\n \"ts\"=\"2026-03-09 10:32:00.092051\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"patch-tags-cluster\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n STEP: getting credentials @ 03/09/26 10:38:03.464\n STEP: ensuring the cluster is viable @ 03/09/26 10:38:54.6\n STEP: sending a PATCH request to set a tag @ 03/09/26 10:38:55.028\n STEP: verifying the tag is present in the update response body @ 03/09/26 10:39:35.857\n STEP: verifying the tag is present on the cluster @ 03/09/26 10:39:35.857\n \"ts\"=\"2026-03-09 10:39:35.967590\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:39:35.967743\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-patch-tags-cluster-ddgqtt\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n \"ts\"=\"2026-03-09 10:39:36.966000\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-patch-tags-cluster-ddgqtt\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:39:38.037822\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n \"ts\"=\"2026-03-09 10:39:38.037962\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n \"ts\"=\"2026-03-09 10:39:38.038046\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:39:38.039324\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-1\"\n \"ts\"=\"2026-03-09 10:39:38.039357\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n \"ts\"=\"2026-03-09 10:39:38.663283\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-1\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:39:39.092513\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-pxqr4n\" \"deployment\"=\"pooledNonMsiScopedAssignments-fklo75htpxzxc\"\n \"ts\"=\"2026-03-09 10:39:39.532248\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-1\" \"deployment\"=\"msi-deployment-1\"\n \"ts\"=\"2026-03-09 10:39:39.990307\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-pxqr4n\" \"deployment\"=\"customer-infra-patch-tags-cluster-ddgqtt\"\n \"ts\"=\"2026-03-09 10:39:40.959434\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:39:40.960089\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:39:40.960324\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n \"ts\"=\"2026-03-09 10:48:35.606888\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n \"ts\"=\"2026-03-09 10:48:35.606954\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"patch-tags-pxqr4n\"\n \"ts\"=\"2026-03-09 10:50:01.392331\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [2353.450 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 2353.451 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"7659f6e99c42c264bf71ca066c4e25bdb815225765a6347b435c0c8299db6afd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"2cdc2112f7241c59079fabbaf3434222f89e9cf9f7a4ed7008dd1d592fb03508","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create node pools with ARM64-based VMs [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n \"ts\"=\"2026-03-09 10:10:47.801070\" \"level\"=0 \"msg\"=\"initialized managed identities pool state\" \"entries\"=20\n STEP: creating a resource group @ 03/09/26 10:10:47.804\n \"ts\"=\"2026-03-09 10:10:47.804626\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n STEP: creating cluster parameters @ 03/09/26 10:10:51.5\n STEP: creating customer resources @ 03/09/26 10:10:51.501\n STEP: creating the HCP cluster @ 03/09/26 10:11:49.665\n \"ts\"=\"2026-03-09 10:11:49.665660\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"arm64-vm-hcp-cluster\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n STEP: discovering an ARM64-capable VM size @ 03/09/26 10:18:24.169\n STEP: creating the ARM64-based node pool \"arm64-vm-np-1\" with VM size \"Standard_D2pls_v6\" @ 03/09/26 10:18:25.11\n \"ts\"=\"2026-03-09 10:25:19.915031\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:25:19.915176\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-h89ts8\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n \"ts\"=\"2026-03-09 10:25:20.978917\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-arm64-vm-hcp-cluster-h89ts8\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:25:21.645813\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n \"ts\"=\"2026-03-09 10:25:21.645947\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n \"ts\"=\"2026-03-09 10:25:21.646032\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:25:21.647012\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-96\"\n \"ts\"=\"2026-03-09 10:25:21.647064\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n \"ts\"=\"2026-03-09 10:25:22.263251\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-96\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:25:22.725482\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\" \"deployment\"=\"pooledNonMsiScopedAssignments-6htyoibevj7ye\"\n \"ts\"=\"2026-03-09 10:25:23.180258\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-96\" \"deployment\"=\"msi-deployment-96\"\n \"ts\"=\"2026-03-09 10:25:23.676997\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-h89ts8\"\n \"ts\"=\"2026-03-09 10:25:24.719240\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:25:24.719426\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:25:24.719569\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n \"ts\"=\"2026-03-09 10:34:08.979543\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n \"ts\"=\"2026-03-09 10:34:08.979599\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n \"ts\"=\"2026-03-09 10:35:39.294939\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1491.509 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1491.509 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create node pools with ARM64-based VMs [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n \"ts\"=\"2026-03-09 10:10:47.801070\" \"level\"=0 \"msg\"=\"initialized managed identities pool state\" \"entries\"=20\n STEP: creating a resource group @ 03/09/26 10:10:47.804\n \"ts\"=\"2026-03-09 10:10:47.804626\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n STEP: creating cluster parameters @ 03/09/26 10:10:51.5\n STEP: creating customer resources @ 03/09/26 10:10:51.501\n STEP: creating the HCP cluster @ 03/09/26 10:11:49.665\n \"ts\"=\"2026-03-09 10:11:49.665660\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"arm64-vm-hcp-cluster\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n STEP: discovering an ARM64-capable VM size @ 03/09/26 10:18:24.169\n STEP: creating the ARM64-based node pool \"arm64-vm-np-1\" with VM size \"Standard_D2pls_v6\" @ 03/09/26 10:18:25.11\n \"ts\"=\"2026-03-09 10:25:19.915031\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:25:19.915176\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-h89ts8\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n \"ts\"=\"2026-03-09 10:25:20.978917\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-arm64-vm-hcp-cluster-h89ts8\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:25:21.645813\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n \"ts\"=\"2026-03-09 10:25:21.645947\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n \"ts\"=\"2026-03-09 10:25:21.646032\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:25:21.647012\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-96\"\n \"ts\"=\"2026-03-09 10:25:21.647064\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n \"ts\"=\"2026-03-09 10:25:22.263251\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-96\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:25:22.725482\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\" \"deployment\"=\"pooledNonMsiScopedAssignments-6htyoibevj7ye\"\n \"ts\"=\"2026-03-09 10:25:23.180258\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-96\" \"deployment\"=\"msi-deployment-96\"\n \"ts\"=\"2026-03-09 10:25:23.676997\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-h89ts8\"\n \"ts\"=\"2026-03-09 10:25:24.719240\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:25:24.719426\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:25:24.719569\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n \"ts\"=\"2026-03-09 10:34:08.979543\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n \"ts\"=\"2026-03-09 10:34:08.979599\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"arm64-vm-cluster-tg2jjn\"\n \"ts\"=\"2026-03-09 10:35:39.294939\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1491.509 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1491.509 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"7fe6862a03d5047064b363ff0753f0d6d482892c5c409e1a118067f0560e4dc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"d9f18428f6cd7eb29f34b02b87dc21bba687734507bfead6e494794bf7126c95","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create an HCP cluster and custom node pool osDisk size [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n STEP: creating a resource group @ 03/09/26 10:30:58.75\n \"ts\"=\"2026-03-09 10:30:58.750300\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 03/09/26 10:31:01.348\n STEP: creating the HCP cluster @ 03/09/26 10:31:49.961\n \"ts\"=\"2026-03-09 10:31:49.961726\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"hcp-cluster-np-128\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n STEP: creating the node pool with custom osDisk size @ 03/09/26 10:37:02.971\n STEP: getting credentials @ 03/09/26 10:43:56.015\n STEP: ensuring the cluster is viable @ 03/09/26 10:44:07.03\n STEP: verifying the node pool is created and has the correct osDisk size @ 03/09/26 10:44:07.42\n \"ts\"=\"2026-03-09 10:44:07.509589\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:44:07.509743\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-hhpt88\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n \"ts\"=\"2026-03-09 10:44:08.781608\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-hcp-cluster-np-128-hhpt88\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:44:09.576694\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n \"ts\"=\"2026-03-09 10:44:09.577016\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n \"ts\"=\"2026-03-09 10:44:09.577253\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:44:09.579335\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-88\"\n \"ts\"=\"2026-03-09 10:44:09.579396\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n \"ts\"=\"2026-03-09 10:44:10.410719\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-88\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:44:11.073515\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-2fbz5q\" \"deployment\"=\"pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\"\n \"ts\"=\"2026-03-09 10:44:11.848324\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-88\" \"deployment\"=\"msi-deployment-88\"\n \"ts\"=\"2026-03-09 10:44:12.551465\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-2fbz5q\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-hhpt88\"\n \"ts\"=\"2026-03-09 10:44:14.032306\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:44:14.032539\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:44:14.032833\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n \"ts\"=\"2026-03-09 10:54:29.109989\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n \"ts\"=\"2026-03-09 10:54:29.110058\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n \"ts\"=\"2026-03-09 10:55:55.441080\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [2707.511 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 2707.512 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create an HCP cluster and custom node pool osDisk size [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n STEP: creating a resource group @ 03/09/26 10:30:58.75\n \"ts\"=\"2026-03-09 10:30:58.750300\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 03/09/26 10:31:01.348\n STEP: creating the HCP cluster @ 03/09/26 10:31:49.961\n \"ts\"=\"2026-03-09 10:31:49.961726\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"hcp-cluster-np-128\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n STEP: creating the node pool with custom osDisk size @ 03/09/26 10:37:02.971\n STEP: getting credentials @ 03/09/26 10:43:56.015\n STEP: ensuring the cluster is viable @ 03/09/26 10:44:07.03\n STEP: verifying the node pool is created and has the correct osDisk size @ 03/09/26 10:44:07.42\n \"ts\"=\"2026-03-09 10:44:07.509589\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:44:07.509743\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-hhpt88\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n \"ts\"=\"2026-03-09 10:44:08.781608\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-hcp-cluster-np-128-hhpt88\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:44:09.576694\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n \"ts\"=\"2026-03-09 10:44:09.577016\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n \"ts\"=\"2026-03-09 10:44:09.577253\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:44:09.579335\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-88\"\n \"ts\"=\"2026-03-09 10:44:09.579396\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n \"ts\"=\"2026-03-09 10:44:10.410719\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-88\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:44:11.073515\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-2fbz5q\" \"deployment\"=\"pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\"\n \"ts\"=\"2026-03-09 10:44:11.848324\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-88\" \"deployment\"=\"msi-deployment-88\"\n \"ts\"=\"2026-03-09 10:44:12.551465\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-2fbz5q\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-hhpt88\"\n \"ts\"=\"2026-03-09 10:44:14.032306\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:44:14.032539\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:44:14.032833\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n \"ts\"=\"2026-03-09 10:54:29.109989\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n \"ts\"=\"2026-03-09 10:54:29.110058\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"clusternp128-2fbz5q\"\n \"ts\"=\"2026-03-09 10:55:55.441080\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [2707.511 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 2707.512 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"816ea0d5a0174ab8f3bc07ddde197f19e57ea003a8d17e3e4fbef3fb14bbca0b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"ddb20e1a8857116da2b896102c1796f608b2989cb8d78cd315879aa8d6e58936","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Engineering should be able to retrieve kusto logs for a cluster and services [PreLaunchSetup:None, Importance:High, Positivity:Positive, Core-Infra-Service, Development-Only, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:34\n STEP: creating a resource group @ 03/09/26 10:10:47.852\n \"ts\"=\"2026-03-09 10:10:47.853001\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n STEP: creating cluster parameters @ 03/09/26 10:10:51.766\n STEP: creating customer resources @ 03/09/26 10:10:51.766\n STEP: creating the HCP cluster @ 03/09/26 10:11:49.181\n \"ts\"=\"2026-03-09 10:11:49.181764\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"engineering-hcp-cluster\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n STEP: verifying kusto logs are present @ 03/09/26 10:18:22.816\n \"ts\"=\"2026-03-09 10:18:31.668019\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:18:31.668366\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-engineering-hcp-cluster-pdnjqw\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n \"ts\"=\"2026-03-09 10:18:32.992820\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-engineering-hcp-cluster-pdnjqw\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:18:33.760631\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-fb6d207289fe1114ce672abf5865312a466e24e7ab0cc85e59e941314773e2c8.yaml.gz\"\n \"ts\"=\"2026-03-09 10:18:33.760894\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-fb6d207289fe1114ce672abf5865312a466e24e7ab0cc85e59e941314773e2c8.yaml.gz\"\n \"ts\"=\"2026-03-09 10:18:33.761243\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:18:33.762830\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-88\"\n \"ts\"=\"2026-03-09 10:18:33.762893\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n \"ts\"=\"2026-03-09 10:18:34.562574\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"kusto-logs-krpv6r\" \"deployment\"=\"pooledNonMsiScopedAssignments-5borhjiq7mwd4\"\n \"ts\"=\"2026-03-09 10:18:35.205550\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-88\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:18:35.863235\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"kusto-logs-krpv6r\" \"deployment\"=\"customer-infra-engineering-hcp-cluster-pdnjqw\"\n \"ts\"=\"2026-03-09 10:18:36.487283\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-88\" \"deployment\"=\"msi-deployment-88\"\n \"ts\"=\"2026-03-09 10:18:37.737071\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:18:37.737270\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:18:37.737436\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n \"ts\"=\"2026-03-09 10:29:02.926275\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n \"ts\"=\"2026-03-09 10:29:02.926358\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n \"ts\"=\"2026-03-09 10:30:40.515645\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1192.687 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1192.688 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Engineering should be able to retrieve kusto logs for a cluster and services [PreLaunchSetup:None, Importance:High, Positivity:Positive, Core-Infra-Service, Development-Only, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:34\n STEP: creating a resource group @ 03/09/26 10:10:47.852\n \"ts\"=\"2026-03-09 10:10:47.853001\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n STEP: creating cluster parameters @ 03/09/26 10:10:51.766\n STEP: creating customer resources @ 03/09/26 10:10:51.766\n STEP: creating the HCP cluster @ 03/09/26 10:11:49.181\n \"ts\"=\"2026-03-09 10:11:49.181764\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"engineering-hcp-cluster\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n STEP: verifying kusto logs are present @ 03/09/26 10:18:22.816\n \"ts\"=\"2026-03-09 10:18:31.668019\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:18:31.668366\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-engineering-hcp-cluster-pdnjqw\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n \"ts\"=\"2026-03-09 10:18:32.992820\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-engineering-hcp-cluster-pdnjqw\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:18:33.760631\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-fb6d207289fe1114ce672abf5865312a466e24e7ab0cc85e59e941314773e2c8.yaml.gz\"\n \"ts\"=\"2026-03-09 10:18:33.760894\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-fb6d207289fe1114ce672abf5865312a466e24e7ab0cc85e59e941314773e2c8.yaml.gz\"\n \"ts\"=\"2026-03-09 10:18:33.761243\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:18:33.762830\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-88\"\n \"ts\"=\"2026-03-09 10:18:33.762893\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n \"ts\"=\"2026-03-09 10:18:34.562574\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"kusto-logs-krpv6r\" \"deployment\"=\"pooledNonMsiScopedAssignments-5borhjiq7mwd4\"\n \"ts\"=\"2026-03-09 10:18:35.205550\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-88\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:18:35.863235\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"kusto-logs-krpv6r\" \"deployment\"=\"customer-infra-engineering-hcp-cluster-pdnjqw\"\n \"ts\"=\"2026-03-09 10:18:36.487283\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-88\" \"deployment\"=\"msi-deployment-88\"\n \"ts\"=\"2026-03-09 10:18:37.737071\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:18:37.737270\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:18:37.737436\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n \"ts\"=\"2026-03-09 10:29:02.926275\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n \"ts\"=\"2026-03-09 10:29:02.926358\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"kusto-logs-krpv6r\"\n \"ts\"=\"2026-03-09 10:30:40.515645\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1192.687 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1192.688 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"86f44e6c9fc1998d3fa19e29c2973c1d7dfbabec5661e238b3812a0ef94b70cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"c770bb35b147e462fe04819b64503d00c1096d5f9a9887c462f3b5d9c9d21a80","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":": creating cluster without node pool: list-test-cluster-x5g4wb @ 03/09/26 10:37:05.684\n STEP: creating customer resources for cluster: list-test-cluster-x5g4wb @ 03/09/26 10:37:05.684\n STEP: creating HCP cluster: list-test-cluster-x5g4wb @ 03/09/26 10:38:20.891\n \"ts\"=\"2026-03-09 10:38:20.891083\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"list-test-cluster-x5g4wb\" \"resourceGroup\"=\"cluster-listing-tj52tb\"\n STEP: testing subscription-level cluster listing @ 03/09/26 10:44:23.565\n STEP: testing resource group-level cluster listing @ 03/09/26 10:44:23.944\n \"ts\"=\"2026-03-09 10:44:24.191537\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:44:24.191706\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-list-test-cluster-ptq6ns-sfxgf7\" \"resourceGroup\"=\"cluster-listing-sztzt2\"\n \"ts\"=\"2026-03-09 10:44:25.481099\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-list-test-cluster-ptq6ns-sfxgf7\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:44:26.684889\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-list-test-cluster-x5g4wb-lb2rp8\" \"resourceGroup\"=\"cluster-listing-tj52tb\"\n \"ts\"=\"2026-03-09 10:44:26.885748\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-list-test-cluster-x5g4wb-lb2rp8\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:44:27.498630\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz\"\n \"ts\"=\"2026-03-09 10:44:27.498776\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz\"\n \"ts\"=\"2026-03-09 10:44:27.498903\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:44:27.500338\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-149\"\n \"ts\"=\"2026-03-09 10:44:27.500424\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-98\"\n \"ts\"=\"2026-03-09 10:44:27.500408\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"cluster-listing-tj52tb\"\n \"ts\"=\"2026-03-09 10:44:27.500483\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"cluster-listing-sztzt2\"\n \"ts\"=\"2026-03-09 10:44:28.189501\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-149\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:44:28.717203\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-98\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:44:29.238430\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-tj52tb\" \"deployment\"=\"pooledNonMsiScopedAssignments-jbjqdczfb6u6g\"\n \"ts\"=\"2026-03-09 10:44:29.821004\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-sztzt2\" \"deployment\"=\"pooledNonMsiScopedAssignments-u4p7fc4nt5fwe\"\n \"ts\"=\"2026-03-09 10:44:30.412003\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-149\" \"deployment\"=\"msi-deployment-149\"\n \"ts\"=\"2026-03-09 10:44:31.005516\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-98\" \"deployment\"=\"msi-deployment-98\"\n \"ts\"=\"2026-03-09 10:44:31.556001\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-tj52tb\" \"deployment\"=\"customer-infra-list-test-cluster-x5g4wb-lb2rp8\"\n \"ts\"=\"2026-03-09 10:44:32.139686\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-sztzt2\" \"deployment\"=\"customer-infra-list-test-cluster-ptq6ns-sfxgf7\"\n \"ts\"=\"2026-03-09 10:44:34.620529\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:44:34.620761\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:44:34.621014\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"cluster-listing-tj52tb\"\n \"ts\"=\"2026-03-09 10:44:34.621129\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"cluster-listing-sztzt2\"\n \"ts\"=\"2026-03-09 10:53:20.240643\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"cluster-listing-tj52tb\"\n \"ts\"=\"2026-03-09 10:53:20.240703\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"cluster-listing-tj52tb\"\n \"ts\"=\"2026-03-09 10:54:29.517335\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"cluster-listing-sztzt2\"\n \"ts\"=\"2026-03-09 10:54:29.517383\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"cluster-listing-sztzt2\"\n \"ts\"=\"2026-03-09 10:56:06.363205\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [2718.487 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 2718.487 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\n: creating cluster without node pool: list-test-cluster-x5g4wb @ 03/09/26 10:37:05.684\n STEP: creating customer resources for cluster: list-test-cluster-x5g4wb @ 03/09/26 10:37:05.684\n STEP: creating HCP cluster: list-test-cluster-x5g4wb @ 03/09/26 10:38:20.891\n \"ts\"=\"2026-03-09 10:38:20.891083\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"list-test-cluster-x5g4wb\" \"resourceGroup\"=\"cluster-listing-tj52tb\"\n STEP: testing subscription-level cluster listing @ 03/09/26 10:44:23.565\n STEP: testing resource group-level cluster listing @ 03/09/26 10:44:23.944\n \"ts\"=\"2026-03-09 10:44:24.191537\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:44:24.191706\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-list-test-cluster-ptq6ns-sfxgf7\" \"resourceGroup\"=\"cluster-listing-sztzt2\"\n \"ts\"=\"2026-03-09 10:44:25.481099\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-list-test-cluster-ptq6ns-sfxgf7\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:44:26.684889\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-list-test-cluster-x5g4wb-lb2rp8\" \"resourceGroup\"=\"cluster-listing-tj52tb\"\n \"ts\"=\"2026-03-09 10:44:26.885748\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-list-test-cluster-x5g4wb-lb2rp8\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:44:27.498630\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz\"\n \"ts\"=\"2026-03-09 10:44:27.498776\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz\"\n \"ts\"=\"2026-03-09 10:44:27.498903\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:44:27.500338\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-149\"\n \"ts\"=\"2026-03-09 10:44:27.500424\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-98\"\n \"ts\"=\"2026-03-09 10:44:27.500408\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"cluster-listing-tj52tb\"\n \"ts\"=\"2026-03-09 10:44:27.500483\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"cluster-listing-sztzt2\"\n \"ts\"=\"2026-03-09 10:44:28.189501\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-149\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:44:28.717203\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-98\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:44:29.238430\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-tj52tb\" \"deployment\"=\"pooledNonMsiScopedAssignments-jbjqdczfb6u6g\"\n \"ts\"=\"2026-03-09 10:44:29.821004\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-sztzt2\" \"deployment\"=\"pooledNonMsiScopedAssignments-u4p7fc4nt5fwe\"\n \"ts\"=\"2026-03-09 10:44:30.412003\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-149\" \"deployment\"=\"msi-deployment-149\"\n \"ts\"=\"2026-03-09 10:44:31.005516\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-98\" \"deployment\"=\"msi-deployment-98\"\n \"ts\"=\"2026-03-09 10:44:31.556001\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-tj52tb\" \"deployment\"=\"customer-infra-list-test-cluster-x5g4wb-lb2rp8\"\n \"ts\"=\"2026-03-09 10:44:32.139686\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-sztzt2\" \"deployment\"=\"customer-infra-list-test-cluster-ptq6ns-sfxgf7\"\n \"ts\"=\"2026-03-09 10:44:34.620529\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:44:34.620761\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:44:34.621014\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"cluster-listing-tj52tb\"\n \"ts\"=\"2026-03-09 10:44:34.621129\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"cluster-listing-sztzt2\"\n \"ts\"=\"2026-03-09 10:53:20.240643\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"cluster-listing-tj52tb\"\n \"ts\"=\"2026-03-09 10:53:20.240703\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"cluster-listing-tj52tb\"\n \"ts\"=\"2026-03-09 10:54:29.517335\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"cluster-listing-sztzt2\"\n \"ts\"=\"2026-03-09 10:54:29.517383\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"cluster-listing-sztzt2\"\n \"ts\"=\"2026-03-09 10:56:06.363205\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [2718.487 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 2718.487 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"b711ca30802291baebf92c8a8dad02cc08a8da46863d1fe404d165fa38186b0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"91cf02cab979f3fa3de2b49ae448505e5b1276a142b27ea2c5dea3ffdf0897ec","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"plan phase is \\\"Installing\\\", expected Complete\\nNo InstallPlan conditions available\"\n STEP: creating NodeFeatureDiscovery CR to deploy NFD worker @ 03/09/26 10:31:05.782\n STEP: waiting for NFD worker DaemonSet to be created @ 03/09/26 10:31:05.853\n STEP: waiting for NFD worker pods to be created and verify images from registry.redhat.io can be pulled @ 03/09/26 10:31:20.961\n \"ts\"=\"2026-03-09 10:31:20.961556\" \"level\"=0 \"msg\"=\"Starting image pull verification\" \"namespace\"=\"openshift-nfd\" \"imageRepository\"=\"registry.redhat.io\" \"imageName\"=\"ose-node-feature-discovery\" \"startTime\"=\"2026-03-09T10:31:20Z\"\n \"ts\"=\"2026-03-09 10:31:21.063281\" \"level\"=0 \"msg\"=\"Container waiting\" \"pod\"=\"nfd-gc-685d666965-pmgj7\" \"container\"=\"nfd-gc\" \"reason\"=\"CreateContainerError\" \"message\"=\"container create failed: executable file `nfd-gc` not found in $PATH: No such file or directory\\n\" \"image\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\"\n \"ts\"=\"2026-03-09 10:31:21.063340\" \"level\"=0 \"msg\"=\"Successfully pulled image\" \"pod\"=\"nfd-master-59c77c96f9-mf5fm\" \"container\"=\"nfd-master\" \"image\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\" \"imageID\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery@sha256:57a132bdf888ea95345027876c952d09ad72f43a5b4488794bf4f6c45f8520da\"\n \"ts\"=\"2026-03-09 10:31:21.063367\" \"level\"=0 \"msg\"=\"Successfully pulled image\" \"pod\"=\"nfd-worker-g5nvf\" \"container\"=\"nfd-worker\" \"image\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\" \"imageID\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery@sha256:57a132bdf888ea95345027876c952d09ad72f43a5b4488794bf4f6c45f8520da\"\n \"ts\"=\"2026-03-09 10:31:21.063404\" \"level\"=0 \"msg\"=\"Successfully pulled image\" \"pod\"=\"nfd-worker-z4lcj\" \"container\"=\"nfd-worker\" \"image\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\" \"imageID\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery@sha256:57a132bdf888ea95345027876c952d09ad72f43a5b4488794bf4f6c45f8520da\"\n \"ts\"=\"2026-03-09 10:31:21.063414\" \"level\"=0 \"msg\"=\"Image pull verification completed successfully\" \"namespace\"=\"openshift-nfd\" \"imageRepository\"=\"registry.redhat.io\" \"imageName\"=\"ose-node-feature-discovery\" \"duration\"=\"101.862899ms\" \"endTime\"=\"2026-03-09T10:31:21Z\"\n \"ts\"=\"2026-03-09 10:31:21.063598\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:31:21.063711\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-pullsecret-hcp-cluster-pnm7dk\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\"\n \"ts\"=\"2026-03-09 10:31:22.198453\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-pullsecret-hcp-cluster-pnm7dk\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:31:22.797001\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a760a96f7a26fb2915b8600ab91b283094b2c3bf2e64d6f8222da667d302697b.yaml.gz\"\n \"ts\"=\"2026-03-09 10:31:22.797202\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a760a96f7a26fb2915b8600ab91b283094b2c3bf2e64d6f8222da667d302697b.yaml.gz\"\n \"ts\"=\"2026-03-09 10:31:22.797291\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:31:22.798495\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-111\"\n \"ts\"=\"2026-03-09 10:31:22.798537\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\"\n \"ts\"=\"2026-03-09 10:31:23.544608\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-111\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:31:23.994258\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\" \"deployment\"=\"pooledNonMsiScopedAssignments-p2h32vyw2l2ma\"\n \"ts\"=\"2026-03-09 10:31:24.616274\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-111\" \"deployment\"=\"msi-deployment-111\"\n \"ts\"=\"2026-03-09 10:31:25.197957\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\" \"deployment\"=\"customer-infra-pullsecret-hcp-cluster-pnm7dk\"\n \"ts\"=\"2026-03-09 10:31:26.294381\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:31:26.294584\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:31:26.294775\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\"\n \"ts\"=\"2026-03-09 10:40:41.310398\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\"\n \"ts\"=\"2026-03-09 10:40:41.310472\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\"\n \"ts\"=\"2026-03-09 10:41:53.715248\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1865.902 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1865.903 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nplan phase is \\\"Installing\\\", expected Complete\\nNo InstallPlan conditions available\"\n STEP: creating NodeFeatureDiscovery CR to deploy NFD worker @ 03/09/26 10:31:05.782\n STEP: waiting for NFD worker DaemonSet to be created @ 03/09/26 10:31:05.853\n STEP: waiting for NFD worker pods to be created and verify images from registry.redhat.io can be pulled @ 03/09/26 10:31:20.961\n \"ts\"=\"2026-03-09 10:31:20.961556\" \"level\"=0 \"msg\"=\"Starting image pull verification\" \"namespace\"=\"openshift-nfd\" \"imageRepository\"=\"registry.redhat.io\" \"imageName\"=\"ose-node-feature-discovery\" \"startTime\"=\"2026-03-09T10:31:20Z\"\n \"ts\"=\"2026-03-09 10:31:21.063281\" \"level\"=0 \"msg\"=\"Container waiting\" \"pod\"=\"nfd-gc-685d666965-pmgj7\" \"container\"=\"nfd-gc\" \"reason\"=\"CreateContainerError\" \"message\"=\"container create failed: executable file `nfd-gc` not found in $PATH: No such file or directory\\n\" \"image\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\"\n \"ts\"=\"2026-03-09 10:31:21.063340\" \"level\"=0 \"msg\"=\"Successfully pulled image\" \"pod\"=\"nfd-master-59c77c96f9-mf5fm\" \"container\"=\"nfd-master\" \"image\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\" \"imageID\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery@sha256:57a132bdf888ea95345027876c952d09ad72f43a5b4488794bf4f6c45f8520da\"\n \"ts\"=\"2026-03-09 10:31:21.063367\" \"level\"=0 \"msg\"=\"Successfully pulled image\" \"pod\"=\"nfd-worker-g5nvf\" \"container\"=\"nfd-worker\" \"image\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\" \"imageID\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery@sha256:57a132bdf888ea95345027876c952d09ad72f43a5b4488794bf4f6c45f8520da\"\n \"ts\"=\"2026-03-09 10:31:21.063404\" \"level\"=0 \"msg\"=\"Successfully pulled image\" \"pod\"=\"nfd-worker-z4lcj\" \"container\"=\"nfd-worker\" \"image\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\" \"imageID\"=\"registry.redhat.io/openshift4/ose-node-feature-discovery@sha256:57a132bdf888ea95345027876c952d09ad72f43a5b4488794bf4f6c45f8520da\"\n \"ts\"=\"2026-03-09 10:31:21.063414\" \"level\"=0 \"msg\"=\"Image pull verification completed successfully\" \"namespace\"=\"openshift-nfd\" \"imageRepository\"=\"registry.redhat.io\" \"imageName\"=\"ose-node-feature-discovery\" \"duration\"=\"101.862899ms\" \"endTime\"=\"2026-03-09T10:31:21Z\"\n \"ts\"=\"2026-03-09 10:31:21.063598\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:31:21.063711\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-pullsecret-hcp-cluster-pnm7dk\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\"\n \"ts\"=\"2026-03-09 10:31:22.198453\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-pullsecret-hcp-cluster-pnm7dk\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:31:22.797001\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a760a96f7a26fb2915b8600ab91b283094b2c3bf2e64d6f8222da667d302697b.yaml.gz\"\n \"ts\"=\"2026-03-09 10:31:22.797202\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a760a96f7a26fb2915b8600ab91b283094b2c3bf2e64d6f8222da667d302697b.yaml.gz\"\n \"ts\"=\"2026-03-09 10:31:22.797291\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:31:22.798495\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-111\"\n \"ts\"=\"2026-03-09 10:31:22.798537\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\"\n \"ts\"=\"2026-03-09 10:31:23.544608\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-111\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:31:23.994258\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\" \"deployment\"=\"pooledNonMsiScopedAssignments-p2h32vyw2l2ma\"\n \"ts\"=\"2026-03-09 10:31:24.616274\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-111\" \"deployment\"=\"msi-deployment-111\"\n \"ts\"=\"2026-03-09 10:31:25.197957\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\" \"deployment\"=\"customer-infra-pullsecret-hcp-cluster-pnm7dk\"\n \"ts\"=\"2026-03-09 10:31:26.294381\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:31:26.294584\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:31:26.294775\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\"\n \"ts\"=\"2026-03-09 10:40:41.310398\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\"\n \"ts\"=\"2026-03-09 10:40:41.310472\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"pullsecret-test-4w8kdv\"\n \"ts\"=\"2026-03-09 10:41:53.715248\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1865.902 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1865.903 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"b9d9eb3fe716fc7f2e180bb28d28960a4f372fb7b2bb6594ab4a25e381c22c3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"9a1655d0ec8d0d51e960c668c85622366ed2d1808b7de44612be5df390c15a8d","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"rectly returned 409 conflict error while cluster is deploying\"\n \"ts\"=\"2026-03-09 10:14:23.458478\" \"level\"=0 \"msg\"=\"Cluster provisioning state updated\" \"provisioningState\"=\"Provisioning\"\n \"ts\"=\"2026-03-09 10:19:24.557785\" \"level\"=0 \"msg\"=\"Cluster provisioning state updated\" \"provisioningState\"=\"Succeeded\"\n STEP: creating 3 admin credentials for the ready cluster @ 03/09/26 10:19:24.557\n STEP: requesting admin credential 1 @ 03/09/26 10:19:24.557\n STEP: validating admin credential 1 works @ 03/09/26 10:20:05.694\n \"ts\"=\"2026-03-09 10:20:05.972578\" \"level\"=0 \"msg\"=\"successfully verified admin credential\" \"credentialNumber\"=1\n STEP: requesting admin credential 2 @ 03/09/26 10:20:05.972\n STEP: validating admin credential 2 works @ 03/09/26 10:20:57.477\n \"ts\"=\"2026-03-09 10:20:57.740284\" \"level\"=0 \"msg\"=\"successfully verified admin credential\" \"credentialNumber\"=2\n STEP: requesting admin credential 3 @ 03/09/26 10:20:57.74\n STEP: validating admin credential 3 works @ 03/09/26 10:21:48.967\n \"ts\"=\"2026-03-09 10:21:49.236550\" \"level\"=0 \"msg\"=\"successfully verified admin credential\" \"credentialNumber\"=3\n STEP: revoking all cluster admin credentials via ARO HCP RP API @ 03/09/26 10:21:49.236\n STEP: validating all admin credentials now fail after revocation @ 03/09/26 10:25:10.781\n STEP: verifying admin credential 1 now fails @ 03/09/26 10:25:10.781\n \"ts\"=\"2026-03-09 10:25:26.066040\" \"level\"=0 \"msg\"=\"successfully verified admin credential fails after revocation\" \"credentialNumber\"=1\n STEP: verifying admin credential 2 now fails @ 03/09/26 10:25:26.066\n \"ts\"=\"2026-03-09 10:25:41.333708\" \"level\"=0 \"msg\"=\"successfully verified admin credential fails after revocation\" \"credentialNumber\"=2\n STEP: verifying admin credential 3 now fails @ 03/09/26 10:25:41.333\n \"ts\"=\"2026-03-09 10:25:56.602453\" \"level\"=0 \"msg\"=\"successfully verified admin credential fails after revocation\" \"credentialNumber\"=3\n STEP: verifying new admin credentials can still be requested after revocation @ 03/09/26 10:25:56.602\n STEP: verifying new admin credentials work after revocation @ 03/09/26 10:26:18.071\n \"ts\"=\"2026-03-09 10:26:18.431687\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:26:18.431817\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-admin-cred-lifecycle-lj6w4s-6zmwvc\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\"\n \"ts\"=\"2026-03-09 10:26:19.414480\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-admin-cred-lifecycle-lj6w4s-6zmwvc\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:26:20.165546\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-1fc3cb04d84cb6ff56f76d767f6559b3ce3af645ad2734354d7330138cf8842a.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:20.165703\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-1fc3cb04d84cb6ff56f76d767f6559b3ce3af645ad2734354d7330138cf8842a.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:20.165816\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:20.166921\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-270\"\n \"ts\"=\"2026-03-09 10:26:20.167041\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\"\n \"ts\"=\"2026-03-09 10:26:20.806873\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-270\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:26:21.265617\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\" \"deployment\"=\"pooledNonMsiScopedAssignments-k537z423zbcxa\"\n \"ts\"=\"2026-03-09 10:26:21.801552\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-270\" \"deployment\"=\"msi-deployment-270\"\n \"ts\"=\"2026-03-09 10:26:22.266037\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\" \"deployment\"=\"customer-infra-admin-cred-lifecycle-lj6w4s-6zmwvc\"\n \"ts\"=\"2026-03-09 10:26:23.393023\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:23.393297\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:26:23.393480\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\"\n \"ts\"=\"2026-03-09 10:36:18.417677\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\"\n \"ts\"=\"2026-03-09 10:36:18.417755\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\"\n \"ts\"=\"2026-03-09 10:37:44.999631\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1617.181 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1617.181 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nrectly returned 409 conflict error while cluster is deploying\"\n \"ts\"=\"2026-03-09 10:14:23.458478\" \"level\"=0 \"msg\"=\"Cluster provisioning state updated\" \"provisioningState\"=\"Provisioning\"\n \"ts\"=\"2026-03-09 10:19:24.557785\" \"level\"=0 \"msg\"=\"Cluster provisioning state updated\" \"provisioningState\"=\"Succeeded\"\n STEP: creating 3 admin credentials for the ready cluster @ 03/09/26 10:19:24.557\n STEP: requesting admin credential 1 @ 03/09/26 10:19:24.557\n STEP: validating admin credential 1 works @ 03/09/26 10:20:05.694\n \"ts\"=\"2026-03-09 10:20:05.972578\" \"level\"=0 \"msg\"=\"successfully verified admin credential\" \"credentialNumber\"=1\n STEP: requesting admin credential 2 @ 03/09/26 10:20:05.972\n STEP: validating admin credential 2 works @ 03/09/26 10:20:57.477\n \"ts\"=\"2026-03-09 10:20:57.740284\" \"level\"=0 \"msg\"=\"successfully verified admin credential\" \"credentialNumber\"=2\n STEP: requesting admin credential 3 @ 03/09/26 10:20:57.74\n STEP: validating admin credential 3 works @ 03/09/26 10:21:48.967\n \"ts\"=\"2026-03-09 10:21:49.236550\" \"level\"=0 \"msg\"=\"successfully verified admin credential\" \"credentialNumber\"=3\n STEP: revoking all cluster admin credentials via ARO HCP RP API @ 03/09/26 10:21:49.236\n STEP: validating all admin credentials now fail after revocation @ 03/09/26 10:25:10.781\n STEP: verifying admin credential 1 now fails @ 03/09/26 10:25:10.781\n \"ts\"=\"2026-03-09 10:25:26.066040\" \"level\"=0 \"msg\"=\"successfully verified admin credential fails after revocation\" \"credentialNumber\"=1\n STEP: verifying admin credential 2 now fails @ 03/09/26 10:25:26.066\n \"ts\"=\"2026-03-09 10:25:41.333708\" \"level\"=0 \"msg\"=\"successfully verified admin credential fails after revocation\" \"credentialNumber\"=2\n STEP: verifying admin credential 3 now fails @ 03/09/26 10:25:41.333\n \"ts\"=\"2026-03-09 10:25:56.602453\" \"level\"=0 \"msg\"=\"successfully verified admin credential fails after revocation\" \"credentialNumber\"=3\n STEP: verifying new admin credentials can still be requested after revocation @ 03/09/26 10:25:56.602\n STEP: verifying new admin credentials work after revocation @ 03/09/26 10:26:18.071\n \"ts\"=\"2026-03-09 10:26:18.431687\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:26:18.431817\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-admin-cred-lifecycle-lj6w4s-6zmwvc\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\"\n \"ts\"=\"2026-03-09 10:26:19.414480\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-admin-cred-lifecycle-lj6w4s-6zmwvc\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:26:20.165546\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-1fc3cb04d84cb6ff56f76d767f6559b3ce3af645ad2734354d7330138cf8842a.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:20.165703\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-1fc3cb04d84cb6ff56f76d767f6559b3ce3af645ad2734354d7330138cf8842a.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:20.165816\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:20.166921\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-270\"\n \"ts\"=\"2026-03-09 10:26:20.167041\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\"\n \"ts\"=\"2026-03-09 10:26:20.806873\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-270\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:26:21.265617\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\" \"deployment\"=\"pooledNonMsiScopedAssignments-k537z423zbcxa\"\n \"ts\"=\"2026-03-09 10:26:21.801552\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-270\" \"deployment\"=\"msi-deployment-270\"\n \"ts\"=\"2026-03-09 10:26:22.266037\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\" \"deployment\"=\"customer-infra-admin-cred-lifecycle-lj6w4s-6zmwvc\"\n \"ts\"=\"2026-03-09 10:26:23.393023\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:23.393297\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:26:23.393480\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\"\n \"ts\"=\"2026-03-09 10:36:18.417677\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\"\n \"ts\"=\"2026-03-09 10:36:18.417755\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"admin-credential-lifecycle-test-sn2hxh\"\n \"ts\"=\"2026-03-09 10:37:44.999631\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1617.181 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1617.181 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"c308540d92cb60a40683b0ad64297df227e28df77ce1bfb2b179b4f37aea5dbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"c510522e1b05903f882ffe7c0c9e0cd1ce611ce269938150efb7f82ffef05329","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to lifecycle and confirm external auth on a cluster [PreLaunchSetup:None, Importance:High, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:43\n STEP: creating resource group for the HCP cluster @ 03/09/26 10:10:47.913\n \"ts\"=\"2026-03-09 10:10:47.914050\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n STEP: creating cluster parameters @ 03/09/26 10:10:52.017\n STEP: creating customer resources @ 03/09/26 10:10:52.017\n STEP: creating HCP cluster @ 03/09/26 10:11:51.327\n \"ts\"=\"2026-03-09 10:11:51.328220\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"ea-listc25m5z\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n STEP: create an external auth and confirm it's in a succeeded state @ 03/09/26 10:18:24.966\n STEP: confirming we're only allowed to create a single external auth @ 03/09/26 10:18:36.608\n STEP: listing all external auth configs to verify a list call works @ 03/09/26 10:18:36.792\n STEP: comparing ARM results with expected external auth config @ 03/09/26 10:18:36.889\n STEP: updating the external auth to a different prefix and confirming the update works @ 03/09/26 10:18:36.889\n \"ts\"=\"2026-03-09 10:18:47.514446\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:18:47.514589\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-ea-listc25m5z-fjp8qc\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n \"ts\"=\"2026-03-09 10:18:48.351279\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-ea-listc25m5z-fjp8qc\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:18:49.146345\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-399a7bed574b7c4dc2699f197c9d42dfc0866e60ee9f0a810da6aa8ecac44a02.yaml.gz\"\n \"ts\"=\"2026-03-09 10:18:49.146642\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-399a7bed574b7c4dc2699f197c9d42dfc0866e60ee9f0a810da6aa8ecac44a02.yaml.gz\"\n \"ts\"=\"2026-03-09 10:18:49.146811\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:18:49.147987\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-203\"\n \"ts\"=\"2026-03-09 10:18:49.148105\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n \"ts\"=\"2026-03-09 10:18:49.865857\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-203\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:18:50.493468\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ea-list-wn5qp9\" \"deployment\"=\"pooledNonMsiScopedAssignments-ay2mmxtcewcro\"\n \"ts\"=\"2026-03-09 10:18:50.976669\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-203\" \"deployment\"=\"msi-deployment-203\"\n \"ts\"=\"2026-03-09 10:18:51.629276\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ea-list-wn5qp9\" \"deployment\"=\"customer-infra-ea-listc25m5z-fjp8qc\"\n \"ts\"=\"2026-03-09 10:18:52.728537\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:18:52.728722\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:18:52.728916\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n \"ts\"=\"2026-03-09 10:28:57.962897\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n \"ts\"=\"2026-03-09 10:28:57.962967\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n \"ts\"=\"2026-03-09 10:30:24.670320\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1176.795 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1176.796 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to lifecycle and confirm external auth on a cluster [PreLaunchSetup:None, Importance:High, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:43\n STEP: creating resource group for the HCP cluster @ 03/09/26 10:10:47.913\n \"ts\"=\"2026-03-09 10:10:47.914050\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n STEP: creating cluster parameters @ 03/09/26 10:10:52.017\n STEP: creating customer resources @ 03/09/26 10:10:52.017\n STEP: creating HCP cluster @ 03/09/26 10:11:51.327\n \"ts\"=\"2026-03-09 10:11:51.328220\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"ea-listc25m5z\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n STEP: create an external auth and confirm it's in a succeeded state @ 03/09/26 10:18:24.966\n STEP: confirming we're only allowed to create a single external auth @ 03/09/26 10:18:36.608\n STEP: listing all external auth configs to verify a list call works @ 03/09/26 10:18:36.792\n STEP: comparing ARM results with expected external auth config @ 03/09/26 10:18:36.889\n STEP: updating the external auth to a different prefix and confirming the update works @ 03/09/26 10:18:36.889\n \"ts\"=\"2026-03-09 10:18:47.514446\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:18:47.514589\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-ea-listc25m5z-fjp8qc\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n \"ts\"=\"2026-03-09 10:18:48.351279\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-ea-listc25m5z-fjp8qc\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:18:49.146345\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-399a7bed574b7c4dc2699f197c9d42dfc0866e60ee9f0a810da6aa8ecac44a02.yaml.gz\"\n \"ts\"=\"2026-03-09 10:18:49.146642\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-399a7bed574b7c4dc2699f197c9d42dfc0866e60ee9f0a810da6aa8ecac44a02.yaml.gz\"\n \"ts\"=\"2026-03-09 10:18:49.146811\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:18:49.147987\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-203\"\n \"ts\"=\"2026-03-09 10:18:49.148105\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n \"ts\"=\"2026-03-09 10:18:49.865857\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-203\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:18:50.493468\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ea-list-wn5qp9\" \"deployment\"=\"pooledNonMsiScopedAssignments-ay2mmxtcewcro\"\n \"ts\"=\"2026-03-09 10:18:50.976669\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-203\" \"deployment\"=\"msi-deployment-203\"\n \"ts\"=\"2026-03-09 10:18:51.629276\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ea-list-wn5qp9\" \"deployment\"=\"customer-infra-ea-listc25m5z-fjp8qc\"\n \"ts\"=\"2026-03-09 10:18:52.728537\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:18:52.728722\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:18:52.728916\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n \"ts\"=\"2026-03-09 10:28:57.962897\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n \"ts\"=\"2026-03-09 10:28:57.962967\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"ea-list-wn5qp9\"\n \"ts\"=\"2026-03-09 10:30:24.670320\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1176.795 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1176.796 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"c595a63fef0196adbf8c81246a9eee4786ddccc34151bb2bfc6e712936ef68f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"2d89bf342a6f2a572dff29630a6e2ef5b6e4a17204f76da2d16a61f1bfe1a35c","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"r\" \"error\"=\"failed to create HCP cluster cluster-nsg-reuse: failed starting cluster creation \\\"cluster-nsg-reuse\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-bpt8vs\\\": PUT https://rp.j2548992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-bpt8vs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nsg-reuse\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Network Security Group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-bpt8vs/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name1' is already in use by another cluster\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n \"ts\"=\"2026-03-09 10:18:28.290255\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:18:28.290383\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-subnet-reuse-45vbfj\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\"\n \"ts\"=\"2026-03-09 10:18:29.058708\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-cluster-65khst\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\"\n \"ts\"=\"2026-03-09 10:18:29.170588\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-cluster-65khst\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:18:29.882612\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-subnet-reuse-45vbfj\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:18:30.638637\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-6950b6d79a3c274e15675866beef4b6fc9a309f9234c1ea3a6f0c13bb6db0b9f.yaml.gz\"\n \"ts\"=\"2026-03-09 10:18:30.638904\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-6950b6d79a3c274e15675866beef4b6fc9a309f9234c1ea3a6f0c13bb6db0b9f.yaml.gz\"\n \"ts\"=\"2026-03-09 10:18:30.639032\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:18:30.640712\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-149\"\n \"ts\"=\"2026-03-09 10:18:30.640723\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\"\n \"ts\"=\"2026-03-09 10:18:30.640766\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-98\"\n \"ts\"=\"2026-03-09 10:18:31.410301\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-149\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:18:32.033938\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-98\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:18:32.679565\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\" \"deployment\"=\"pooledNonMsiScopedAssignments-67tmrjj4do2cc\"\n \"ts\"=\"2026-03-09 10:18:33.275648\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-149\" \"deployment\"=\"msi-deployment-149\"\n \"ts\"=\"2026-03-09 10:18:33.763144\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-98\" \"deployment\"=\"msi-deployment-98\"\n \"ts\"=\"2026-03-09 10:18:34.410448\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\" \"deployment\"=\"pooledNonMsiScopedAssignments-j3gkygvtvkvjs\"\n \"ts\"=\"2026-03-09 10:18:36.209875\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\" \"deployment\"=\"customer-infra-basic-cluster-65khst\"\n \"ts\"=\"2026-03-09 10:18:36.959561\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\" \"deployment\"=\"customer-infra-cluster-subnet-reuse-45vbfj\"\n \"ts\"=\"2026-03-09 10:18:37.924952\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:18:37.925145\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:18:37.925329\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\"\n \"ts\"=\"2026-03-09 10:28:02.381628\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\"\n \"ts\"=\"2026-03-09 10:28:02.381678\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\"\n \"ts\"=\"2026-03-09 10:29:36.898827\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1129.013 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1129.013 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nr\" \"error\"=\"failed to create HCP cluster cluster-nsg-reuse: failed starting cluster creation \\\"cluster-nsg-reuse\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-bpt8vs\\\": PUT https://rp.j2548992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-bpt8vs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nsg-reuse\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Network Security Group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-bpt8vs/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name1' is already in use by another cluster\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n \"ts\"=\"2026-03-09 10:18:28.290255\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:18:28.290383\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-subnet-reuse-45vbfj\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\"\n \"ts\"=\"2026-03-09 10:18:29.058708\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-cluster-65khst\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\"\n \"ts\"=\"2026-03-09 10:18:29.170588\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-cluster-65khst\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:18:29.882612\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-subnet-reuse-45vbfj\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:18:30.638637\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-6950b6d79a3c274e15675866beef4b6fc9a309f9234c1ea3a6f0c13bb6db0b9f.yaml.gz\"\n \"ts\"=\"2026-03-09 10:18:30.638904\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-6950b6d79a3c274e15675866beef4b6fc9a309f9234c1ea3a6f0c13bb6db0b9f.yaml.gz\"\n \"ts\"=\"2026-03-09 10:18:30.639032\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:18:30.640712\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-149\"\n \"ts\"=\"2026-03-09 10:18:30.640723\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\"\n \"ts\"=\"2026-03-09 10:18:30.640766\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-98\"\n \"ts\"=\"2026-03-09 10:18:31.410301\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-149\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:18:32.033938\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-98\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:18:32.679565\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\" \"deployment\"=\"pooledNonMsiScopedAssignments-67tmrjj4do2cc\"\n \"ts\"=\"2026-03-09 10:18:33.275648\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-149\" \"deployment\"=\"msi-deployment-149\"\n \"ts\"=\"2026-03-09 10:18:33.763144\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-98\" \"deployment\"=\"msi-deployment-98\"\n \"ts\"=\"2026-03-09 10:18:34.410448\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\" \"deployment\"=\"pooledNonMsiScopedAssignments-j3gkygvtvkvjs\"\n \"ts\"=\"2026-03-09 10:18:36.209875\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\" \"deployment\"=\"customer-infra-basic-cluster-65khst\"\n \"ts\"=\"2026-03-09 10:18:36.959561\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\" \"deployment\"=\"customer-infra-cluster-subnet-reuse-45vbfj\"\n \"ts\"=\"2026-03-09 10:18:37.924952\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:18:37.925145\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:18:37.925329\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\"\n \"ts\"=\"2026-03-09 10:28:02.381628\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\"\n \"ts\"=\"2026-03-09 10:28:02.381678\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-bpt8vs\"\n \"ts\"=\"2026-03-09 10:29:36.898827\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1129.013 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1129.013 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"c665787c85f39bf44c0cf35e1e260c20c3c90e877ab510dfcb66c33d2304fcda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"90391bcb9c3f3579f5c81b4666cf7f08632e429ddbaa9595d861c748ffba3a7b","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create an HCP cluster with Image Registry not present [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:37\n STEP: creating a resource group @ 03/09/26 10:10:47.846\n \"ts\"=\"2026-03-09 10:10:47.846846\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n STEP: creating cluster parameters @ 03/09/26 10:10:51.715\n STEP: creating customer resources @ 03/09/26 10:10:51.715\n STEP: creating the hcp cluster with the image registry disabled @ 03/09/26 10:11:40.481\n \"ts\"=\"2026-03-09 10:11:40.481447\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"disabled-image-registry-hcp-cluster\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n STEP: getting credentials @ 03/09/26 10:19:25.198\n \"ts\"=\"2026-03-09 10:20:08.134282\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:20:08.134408\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-disabled-image-registry-hcp-cluster-nr6j4l\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n \"ts\"=\"2026-03-09 10:20:09.348970\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-disabled-image-registry-hcp-cluster-nr6j4l\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:20:10.522949\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-f66366cb92003bba305ea6fa81a0011f709bf02a764a23c42e292ad6f9754eee.yaml.gz\"\n \"ts\"=\"2026-03-09 10:20:10.523069\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-f66366cb92003bba305ea6fa81a0011f709bf02a764a23c42e292ad6f9754eee.yaml.gz\"\n \"ts\"=\"2026-03-09 10:20:10.523157\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:20:10.524439\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-7\"\n \"ts\"=\"2026-03-09 10:20:10.524485\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n \"ts\"=\"2026-03-09 10:20:11.188104\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-7\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:20:11.573893\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\" \"deployment\"=\"pooledNonMsiScopedAssignments-qmh74zogaodhg\"\n \"ts\"=\"2026-03-09 10:20:12.302658\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-7\" \"deployment\"=\"msi-deployment-7\"\n \"ts\"=\"2026-03-09 10:20:12.550050\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\" \"deployment\"=\"customer-infra-disabled-image-registry-hcp-cluster-nr6j4l\"\n \"ts\"=\"2026-03-09 10:20:13.526428\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:20:13.526614\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:20:13.526761\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n \"ts\"=\"2026-03-09 10:28:58.626840\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n \"ts\"=\"2026-03-09 10:28:58.626992\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n \"ts\"=\"2026-03-09 10:30:28.961769\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1181.122 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1181.122 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create an HCP cluster with Image Registry not present [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:37\n STEP: creating a resource group @ 03/09/26 10:10:47.846\n \"ts\"=\"2026-03-09 10:10:47.846846\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n STEP: creating cluster parameters @ 03/09/26 10:10:51.715\n STEP: creating customer resources @ 03/09/26 10:10:51.715\n STEP: creating the hcp cluster with the image registry disabled @ 03/09/26 10:11:40.481\n \"ts\"=\"2026-03-09 10:11:40.481447\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"disabled-image-registry-hcp-cluster\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n STEP: getting credentials @ 03/09/26 10:19:25.198\n \"ts\"=\"2026-03-09 10:20:08.134282\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:20:08.134408\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-disabled-image-registry-hcp-cluster-nr6j4l\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n \"ts\"=\"2026-03-09 10:20:09.348970\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-disabled-image-registry-hcp-cluster-nr6j4l\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:20:10.522949\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-f66366cb92003bba305ea6fa81a0011f709bf02a764a23c42e292ad6f9754eee.yaml.gz\"\n \"ts\"=\"2026-03-09 10:20:10.523069\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-f66366cb92003bba305ea6fa81a0011f709bf02a764a23c42e292ad6f9754eee.yaml.gz\"\n \"ts\"=\"2026-03-09 10:20:10.523157\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:20:10.524439\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-7\"\n \"ts\"=\"2026-03-09 10:20:10.524485\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n \"ts\"=\"2026-03-09 10:20:11.188104\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-7\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:20:11.573893\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\" \"deployment\"=\"pooledNonMsiScopedAssignments-qmh74zogaodhg\"\n \"ts\"=\"2026-03-09 10:20:12.302658\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-7\" \"deployment\"=\"msi-deployment-7\"\n \"ts\"=\"2026-03-09 10:20:12.550050\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\" \"deployment\"=\"customer-infra-disabled-image-registry-hcp-cluster-nr6j4l\"\n \"ts\"=\"2026-03-09 10:20:13.526428\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:20:13.526614\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:20:13.526761\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n \"ts\"=\"2026-03-09 10:28:58.626840\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n \"ts\"=\"2026-03-09 10:28:58.626992\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"disabled-image-registry-g9p7wd\"\n \"ts\"=\"2026-03-09 10:30:28.961769\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1181.122 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1181.122 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"ccde530fcfde3daaf2902b5bf437267f4d3c81610e422ad3696a8a10349c05ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"a6f813536970a8aa42fb912026aaddb014872a9f987f9bdbb21a6f567eff7673","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update nodepool replicas and autoscaling [PreLaunchSetup:None, Importance:High, Positivity:Positive, ARO-HCP-RP-API-Compatible, Speed:Slow]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:35\n STEP: creating a resource group @ 03/09/26 10:30:58.632\n \"ts\"=\"2026-03-09 10:30:58.632647\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n STEP: creating cluster parameters @ 03/09/26 10:31:01.566\n STEP: creating customer resources @ 03/09/26 10:31:01.566\n STEP: creating the HCP cluster @ 03/09/26 10:31:59.488\n \"ts\"=\"2026-03-09 10:31:59.488288\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-update-nodes-hcp-cluster\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n STEP: getting admin credentials for the cluster @ 03/09/26 10:38:53.171\n STEP: creating the node pools in parallel @ 03/09/26 10:39:34.897\n STEP: verifying nodes count and status after initial creation @ 03/09/26 10:48:48.949\n STEP: scaling up the nodepool replicas from 2 to 3 replicas @ 03/09/26 10:48:49.365\n STEP: verifying nodes count and status after scaling up @ 03/09/26 10:55:42.402\n STEP: scaling down the nodepool replicas from 3 to 2 replicas @ 03/09/26 10:55:42.864\n STEP: verifying nodes count and status after scaling down @ 03/09/26 11:00:25.346\n STEP: updating the one-replica nodepool replicas to 0 and enabling autoscaling with a PATCH @ 03/09/26 11:00:25.791\n STEP: verifying nodes count and status after enabling autoscaling @ 03/09/26 11:06:59.517\n \"ts\"=\"2026-03-09 11:06:59.944387\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 11:06:59.944778\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-update-nodes-hcp-cluster-b8v7wz\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n \"ts\"=\"2026-03-09 11:07:01.436736\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-update-nodes-hcp-cluster-b8v7wz\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 11:07:02.564999\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4d7111da7878873ab737ec2a993f733d74239d4c919cccd4b38c36a47c0a2edd.yaml.gz\"\n \"ts\"=\"2026-03-09 11:07:02.565206\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4d7111da7878873ab737ec2a993f733d74239d4c919cccd4b38c36a47c0a2edd.yaml.gz\"\n \"ts\"=\"2026-03-09 11:07:02.565440\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 11:07:02.568263\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-7\"\n \"ts\"=\"2026-03-09 11:07:02.568360\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n \"ts\"=\"2026-03-09 11:07:03.417030\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-7\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 11:07:03.810298\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\" \"deployment\"=\"pooledNonMsiScopedAssignments-w3k44uxsb3qey\"\n \"ts\"=\"2026-03-09 11:07:04.354472\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-7\" \"deployment\"=\"msi-deployment-7\"\n \"ts\"=\"2026-03-09 11:07:04.842753\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\" \"deployment\"=\"customer-infra-np-update-nodes-hcp-cluster-b8v7wz\"\n \"ts\"=\"2026-03-09 11:07:06.013930\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 11:07:06.015009\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 11:07:06.015446\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n \"ts\"=\"2026-03-09 11:16:30.916136\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n \"ts\"=\"2026-03-09 11:16:30.916215\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n \"ts\"=\"2026-03-09 11:17:42.284327\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [4014.371 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 4014.372 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update nodepool replicas and autoscaling [PreLaunchSetup:None, Importance:High, Positivity:Positive, ARO-HCP-RP-API-Compatible, Speed:Slow]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:35\n STEP: creating a resource group @ 03/09/26 10:30:58.632\n \"ts\"=\"2026-03-09 10:30:58.632647\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n STEP: creating cluster parameters @ 03/09/26 10:31:01.566\n STEP: creating customer resources @ 03/09/26 10:31:01.566\n STEP: creating the HCP cluster @ 03/09/26 10:31:59.488\n \"ts\"=\"2026-03-09 10:31:59.488288\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-update-nodes-hcp-cluster\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n STEP: getting admin credentials for the cluster @ 03/09/26 10:38:53.171\n STEP: creating the node pools in parallel @ 03/09/26 10:39:34.897\n STEP: verifying nodes count and status after initial creation @ 03/09/26 10:48:48.949\n STEP: scaling up the nodepool replicas from 2 to 3 replicas @ 03/09/26 10:48:49.365\n STEP: verifying nodes count and status after scaling up @ 03/09/26 10:55:42.402\n STEP: scaling down the nodepool replicas from 3 to 2 replicas @ 03/09/26 10:55:42.864\n STEP: verifying nodes count and status after scaling down @ 03/09/26 11:00:25.346\n STEP: updating the one-replica nodepool replicas to 0 and enabling autoscaling with a PATCH @ 03/09/26 11:00:25.791\n STEP: verifying nodes count and status after enabling autoscaling @ 03/09/26 11:06:59.517\n \"ts\"=\"2026-03-09 11:06:59.944387\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 11:06:59.944778\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-update-nodes-hcp-cluster-b8v7wz\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n \"ts\"=\"2026-03-09 11:07:01.436736\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-update-nodes-hcp-cluster-b8v7wz\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 11:07:02.564999\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4d7111da7878873ab737ec2a993f733d74239d4c919cccd4b38c36a47c0a2edd.yaml.gz\"\n \"ts\"=\"2026-03-09 11:07:02.565206\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4d7111da7878873ab737ec2a993f733d74239d4c919cccd4b38c36a47c0a2edd.yaml.gz\"\n \"ts\"=\"2026-03-09 11:07:02.565440\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 11:07:02.568263\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-7\"\n \"ts\"=\"2026-03-09 11:07:02.568360\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n \"ts\"=\"2026-03-09 11:07:03.417030\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-7\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 11:07:03.810298\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\" \"deployment\"=\"pooledNonMsiScopedAssignments-w3k44uxsb3qey\"\n \"ts\"=\"2026-03-09 11:07:04.354472\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-7\" \"deployment\"=\"msi-deployment-7\"\n \"ts\"=\"2026-03-09 11:07:04.842753\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\" \"deployment\"=\"customer-infra-np-update-nodes-hcp-cluster-b8v7wz\"\n \"ts\"=\"2026-03-09 11:07:06.013930\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 11:07:06.015009\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 11:07:06.015446\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n \"ts\"=\"2026-03-09 11:16:30.916136\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n \"ts\"=\"2026-03-09 11:16:30.916215\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"nodepool-update-nodes-2hkwzb\"\n \"ts\"=\"2026-03-09 11:17:42.284327\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [4014.371 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 4014.372 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"d0f360c4dd396bc6481157ee166fde60741c0ced04ca9fefd8ce961b2f6140bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"620412029e36909e9bb6fd5be9ecedf1595cccd41b9cffa413e92c74e848dde4","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create an HCP cluster with custom autoscaling [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:36\n STEP: creating a resource group @ 03/09/26 10:10:47.871\n \"ts\"=\"2026-03-09 10:10:47.871225\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n STEP: creating customer resources @ 03/09/26 10:10:52.131\n STEP: creating the cluster @ 03/09/26 10:11:49.528\n \"ts\"=\"2026-03-09 10:11:49.528704\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"autoscaling-hcp-cluster\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n STEP: ensuring the custom autoscaling was honored @ 03/09/26 10:18:23.022\n STEP: creating the node pool @ 03/09/26 10:18:23.123\n STEP: patching the cluster to set maxNodesTotal @ 03/09/26 10:25:36.123\n \"ts\"=\"2026-03-09 10:26:06.967601\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:26:06.967744\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-autoscaling-hcp-cluster-rsq687\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n \"ts\"=\"2026-03-09 10:26:07.974257\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-autoscaling-hcp-cluster-rsq687\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:26:08.667626\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-31a71a35884d87a4a1de36eceeb324011e1c775154472da33ffc40cde64c08cb.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:08.667748\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-31a71a35884d87a4a1de36eceeb324011e1c775154472da33ffc40cde64c08cb.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:08.667828\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:08.668858\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-120\"\n \"ts\"=\"2026-03-09 10:26:08.668902\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n \"ts\"=\"2026-03-09 10:26:09.302296\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-120\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:26:09.765368\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\" \"deployment\"=\"pooledNonMsiScopedAssignments-e4ixwyxsfjnje\"\n \"ts\"=\"2026-03-09 10:26:10.282160\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-120\" \"deployment\"=\"msi-deployment-120\"\n \"ts\"=\"2026-03-09 10:26:10.729734\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\" \"deployment\"=\"customer-infra-autoscaling-hcp-cluster-rsq687\"\n \"ts\"=\"2026-03-09 10:26:11.723104\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:11.723310\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:26:11.723489\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n \"ts\"=\"2026-03-09 10:36:56.784054\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n \"ts\"=\"2026-03-09 10:36:56.784111\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n \"ts\"=\"2026-03-09 10:38:22.700123\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1654.878 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1654.879 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create an HCP cluster with custom autoscaling [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:36\n STEP: creating a resource group @ 03/09/26 10:10:47.871\n \"ts\"=\"2026-03-09 10:10:47.871225\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n STEP: creating customer resources @ 03/09/26 10:10:52.131\n STEP: creating the cluster @ 03/09/26 10:11:49.528\n \"ts\"=\"2026-03-09 10:11:49.528704\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"autoscaling-hcp-cluster\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n STEP: ensuring the custom autoscaling was honored @ 03/09/26 10:18:23.022\n STEP: creating the node pool @ 03/09/26 10:18:23.123\n STEP: patching the cluster to set maxNodesTotal @ 03/09/26 10:25:36.123\n \"ts\"=\"2026-03-09 10:26:06.967601\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:26:06.967744\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-autoscaling-hcp-cluster-rsq687\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n \"ts\"=\"2026-03-09 10:26:07.974257\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-autoscaling-hcp-cluster-rsq687\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:26:08.667626\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-31a71a35884d87a4a1de36eceeb324011e1c775154472da33ffc40cde64c08cb.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:08.667748\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-31a71a35884d87a4a1de36eceeb324011e1c775154472da33ffc40cde64c08cb.yaml.gz\"\n \"ts\"=\"2026-03-09 10:26:08.667828\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:08.668858\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-120\"\n \"ts\"=\"2026-03-09 10:26:08.668902\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n \"ts\"=\"2026-03-09 10:26:09.302296\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-120\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:26:09.765368\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\" \"deployment\"=\"pooledNonMsiScopedAssignments-e4ixwyxsfjnje\"\n \"ts\"=\"2026-03-09 10:26:10.282160\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-120\" \"deployment\"=\"msi-deployment-120\"\n \"ts\"=\"2026-03-09 10:26:10.729734\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\" \"deployment\"=\"customer-infra-autoscaling-hcp-cluster-rsq687\"\n \"ts\"=\"2026-03-09 10:26:11.723104\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:26:11.723310\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:26:11.723489\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n \"ts\"=\"2026-03-09 10:36:56.784054\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n \"ts\"=\"2026-03-09 10:36:56.784111\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"autoscaling-cluster-sk2ph2\"\n \"ts\"=\"2026-03-09 10:38:22.700123\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1654.878 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1654.879 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"d555352e8ad27d17b129b59741c555f269319ba082ade2ecf48719bd38b0b61c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"62792fb33ab0811d5b478b7bf6a2c80afa36ff46372256ea978ad7f86274ca19","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"-connectivity-test\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n STEP: getting cluster details @ 03/09/26 10:19:27.349\n STEP: verifying authorized CIDRs contains VM IP @ 03/09/26 10:19:27.456\n STEP: testing connectivity from authorized VM @ 03/09/26 10:19:27.456\n STEP: VM received HTTP status code: 200 @ 03/09/26 10:19:59.766\n STEP: testing connectivity from current machine (should be blocked) @ 03/09/26 10:19:59.766\n STEP: verifying VM can access cluster API with credentials @ 03/09/26 10:19:59.923\n STEP: kubectl command executed from authorized VM @ 03/09/26 10:21:22.297\n STEP: Successfully retrieved node list with 0 items @ 03/09/26 10:21:22.298\n STEP: verifying aggregated API services from authorized VM @ 03/09/26 10:21:22.298\n STEP: creating the node pool @ 03/09/26 10:21:53.737\n STEP: creating an app registration with a client secret @ 03/09/26 10:30:07.403\n \"ts\"=\"2026-03-09 10:30:07.403848\" \"level\"=0 \"msg\"=\"creating app registration\" \"appName\"=\"aro-hcp-e2e-5476705466293221631\"\n STEP: creating an external auth config with a prefix @ 03/09/26 10:30:10.775\n STEP: verifying ExternalAuth is in a Succeeded state @ 03/09/26 10:30:21.402\n STEP: creating a cluster role binding for the entra application via VM @ 03/09/26 10:30:21.484\n STEP: creating a rest config using OIDC authentication @ 03/09/26 10:30:52.984\n STEP: Successfully retrieved 2 nodes using external auth @ 03/09/26 10:33:07.671\n STEP: creating the console OAuth client secret for external auth via VM @ 03/09/26 10:33:07.671\n STEP: verifying all cluster operators are healthy from authorized VM @ 03/09/26 10:33:39.175\n STEP: updating cluster to remove VM from authorized CIDRs @ 03/09/26 10:34:10.768\n STEP: verifying VM is now blocked from API access @ 03/09/26 10:35:41.728\n STEP: Connection correctly blocked after removing VM from authorized CIDRs @ 03/09/26 10:36:13.066\n \"ts\"=\"2026-03-09 10:36:13.066897\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:36:13.067042\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cidr-connectivity-test-pqrp5j\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n \"ts\"=\"2026-03-09 10:36:14.536029\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cidr-connectivity-test-pqrp5j\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:36:15.233877\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"test-vm\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n \"ts\"=\"2026-03-09 10:36:15.351920\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-5585d0232685e8d5bbce4642dbb894d73a2597e55ff59b2a6006f2460a217da1.yaml.gz\"\n \"ts\"=\"2026-03-09 10:36:15.352077\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-5585d0232685e8d5bbce4642dbb894d73a2597e55ff59b2a6006f2460a217da1.yaml.gz\"\n \"ts\"=\"2026-03-09 10:36:15.352214\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:36:15.354156\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-193\"\n \"ts\"=\"2026-03-09 10:36:15.354205\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n \"ts\"=\"2026-03-09 10:36:16.041965\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-193\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:36:16.625752\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\" \"deployment\"=\"test-vm\"\n \"ts\"=\"2026-03-09 10:36:17.200292\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-193\" \"deployment\"=\"msi-deployment-193\"\n \"ts\"=\"2026-03-09 10:36:17.846350\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\" \"deployment\"=\"pooledNonMsiScopedAssignments-wihoigd7cpguy\"\n \"ts\"=\"2026-03-09 10:36:19.004449\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\" \"deployment\"=\"customer-infra-cidr-connectivity-test-pqrp5j\"\n \"ts\"=\"2026-03-09 10:36:19.724799\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:36:19.725051\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:36:19.725300\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n \"ts\"=\"2026-03-09 10:45:14.365122\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n \"ts\"=\"2026-03-09 10:45:14.365180\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n \"ts\"=\"2026-03-09 10:48:29.310761\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [2261.432 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 2261.433 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\n-connectivity-test\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n STEP: getting cluster details @ 03/09/26 10:19:27.349\n STEP: verifying authorized CIDRs contains VM IP @ 03/09/26 10:19:27.456\n STEP: testing connectivity from authorized VM @ 03/09/26 10:19:27.456\n STEP: VM received HTTP status code: 200 @ 03/09/26 10:19:59.766\n STEP: testing connectivity from current machine (should be blocked) @ 03/09/26 10:19:59.766\n STEP: verifying VM can access cluster API with credentials @ 03/09/26 10:19:59.923\n STEP: kubectl command executed from authorized VM @ 03/09/26 10:21:22.297\n STEP: Successfully retrieved node list with 0 items @ 03/09/26 10:21:22.298\n STEP: verifying aggregated API services from authorized VM @ 03/09/26 10:21:22.298\n STEP: creating the node pool @ 03/09/26 10:21:53.737\n STEP: creating an app registration with a client secret @ 03/09/26 10:30:07.403\n \"ts\"=\"2026-03-09 10:30:07.403848\" \"level\"=0 \"msg\"=\"creating app registration\" \"appName\"=\"aro-hcp-e2e-5476705466293221631\"\n STEP: creating an external auth config with a prefix @ 03/09/26 10:30:10.775\n STEP: verifying ExternalAuth is in a Succeeded state @ 03/09/26 10:30:21.402\n STEP: creating a cluster role binding for the entra application via VM @ 03/09/26 10:30:21.484\n STEP: creating a rest config using OIDC authentication @ 03/09/26 10:30:52.984\n STEP: Successfully retrieved 2 nodes using external auth @ 03/09/26 10:33:07.671\n STEP: creating the console OAuth client secret for external auth via VM @ 03/09/26 10:33:07.671\n STEP: verifying all cluster operators are healthy from authorized VM @ 03/09/26 10:33:39.175\n STEP: updating cluster to remove VM from authorized CIDRs @ 03/09/26 10:34:10.768\n STEP: verifying VM is now blocked from API access @ 03/09/26 10:35:41.728\n STEP: Connection correctly blocked after removing VM from authorized CIDRs @ 03/09/26 10:36:13.066\n \"ts\"=\"2026-03-09 10:36:13.066897\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:36:13.067042\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cidr-connectivity-test-pqrp5j\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n \"ts\"=\"2026-03-09 10:36:14.536029\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cidr-connectivity-test-pqrp5j\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:36:15.233877\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"test-vm\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n \"ts\"=\"2026-03-09 10:36:15.351920\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-5585d0232685e8d5bbce4642dbb894d73a2597e55ff59b2a6006f2460a217da1.yaml.gz\"\n \"ts\"=\"2026-03-09 10:36:15.352077\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-5585d0232685e8d5bbce4642dbb894d73a2597e55ff59b2a6006f2460a217da1.yaml.gz\"\n \"ts\"=\"2026-03-09 10:36:15.352214\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:36:15.354156\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-193\"\n \"ts\"=\"2026-03-09 10:36:15.354205\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n \"ts\"=\"2026-03-09 10:36:16.041965\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-193\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:36:16.625752\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\" \"deployment\"=\"test-vm\"\n \"ts\"=\"2026-03-09 10:36:17.200292\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-193\" \"deployment\"=\"msi-deployment-193\"\n \"ts\"=\"2026-03-09 10:36:17.846350\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\" \"deployment\"=\"pooledNonMsiScopedAssignments-wihoigd7cpguy\"\n \"ts\"=\"2026-03-09 10:36:19.004449\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\" \"deployment\"=\"customer-infra-cidr-connectivity-test-pqrp5j\"\n \"ts\"=\"2026-03-09 10:36:19.724799\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:36:19.725051\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:36:19.725300\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n \"ts\"=\"2026-03-09 10:45:14.365122\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n \"ts\"=\"2026-03-09 10:45:14.365180\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"e2e-cidr-connectivity-9m625b\"\n \"ts\"=\"2026-03-09 10:48:29.310761\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [2261.432 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 2261.433 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}},{"row_id":"efd565194dafec8839d09b209513534e21f45573db9968fc18933621de6af15c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030942918222548992","occurred_at":"2026-03-09T09:44:59Z","failed_at":"e2e","signature_id":"12c80b3c34165ae408f8d00407d8014bda9b60008b76be2789c9f21612bdd90a","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should respect cluster-wide node limits with nodepool autoscaling [PreLaunchSetup:None, Importance:Medium, Positivity:Negative, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122\n STEP: creating a resource group @ 03/09/26 10:10:47.883\n \"ts\"=\"2026-03-09 10:10:47.883143\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n STEP: creating cluster with MaxNodesTotal limit @ 03/09/26 10:10:52.983\n \"ts\"=\"2026-03-09 10:11:49.750674\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"node-limit-cluster\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n STEP: attempting to create nodepool with Min \u003e MaxNodesTotal @ 03/09/26 10:20:15.181\n \"ts\"=\"2026-03-09 10:20:15.312648\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:20:15.312768\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-node-limit-cluster-nswvmq\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n \"ts\"=\"2026-03-09 10:20:16.325689\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-node-limit-cluster-nswvmq\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:20:17.593532\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-133f4b3b85cd881bd2d7025c4df814bf698b577d4462803137df4f5ac0cf68e9.yaml.gz\"\n \"ts\"=\"2026-03-09 10:20:17.593647\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-133f4b3b85cd881bd2d7025c4df814bf698b577d4462803137df4f5ac0cf68e9.yaml.gz\"\n \"ts\"=\"2026-03-09 10:20:17.593734\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:20:17.594708\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-130\"\n \"ts\"=\"2026-03-09 10:20:17.594745\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n \"ts\"=\"2026-03-09 10:20:18.244092\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-130\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:20:18.683805\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"node-limit-test-rmm4pp\" \"deployment\"=\"pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\"\n \"ts\"=\"2026-03-09 10:20:19.205277\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-130\" \"deployment\"=\"msi-deployment-130\"\n \"ts\"=\"2026-03-09 10:20:19.637290\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"node-limit-test-rmm4pp\" \"deployment\"=\"customer-infra-node-limit-cluster-nswvmq\"\n \"ts\"=\"2026-03-09 10:20:20.677395\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:20:20.677659\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:20:20.677981\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n \"ts\"=\"2026-03-09 10:30:05.795084\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n \"ts\"=\"2026-03-09 10:30:05.795256\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n \"ts\"=\"2026-03-09 10:31:17.926615\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1230.077 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1230.077 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1773051047 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should respect cluster-wide node limits with nodepool autoscaling [PreLaunchSetup:None, Importance:Medium, Positivity:Negative, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122\n STEP: creating a resource group @ 03/09/26 10:10:47.883\n \"ts\"=\"2026-03-09 10:10:47.883143\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n STEP: creating cluster with MaxNodesTotal limit @ 03/09/26 10:10:52.983\n \"ts\"=\"2026-03-09 10:11:49.750674\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"node-limit-cluster\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n STEP: attempting to create nodepool with Min \u003e MaxNodesTotal @ 03/09/26 10:20:15.181\n \"ts\"=\"2026-03-09 10:20:15.312648\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-09 10:20:15.312768\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-node-limit-cluster-nswvmq\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n \"ts\"=\"2026-03-09 10:20:16.325689\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-node-limit-cluster-nswvmq\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-09 10:20:17.593532\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-133f4b3b85cd881bd2d7025c4df814bf698b577d4462803137df4f5ac0cf68e9.yaml.gz\"\n \"ts\"=\"2026-03-09 10:20:17.593647\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-133f4b3b85cd881bd2d7025c4df814bf698b577d4462803137df4f5ac0cf68e9.yaml.gz\"\n \"ts\"=\"2026-03-09 10:20:17.593734\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-09 10:20:17.594708\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-130\"\n \"ts\"=\"2026-03-09 10:20:17.594745\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n \"ts\"=\"2026-03-09 10:20:18.244092\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-130\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-09 10:20:18.683805\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"node-limit-test-rmm4pp\" \"deployment\"=\"pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\"\n \"ts\"=\"2026-03-09 10:20:19.205277\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-130\" \"deployment\"=\"msi-deployment-130\"\n \"ts\"=\"2026-03-09 10:20:19.637290\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"node-limit-test-rmm4pp\" \"deployment\"=\"customer-infra-node-limit-cluster-nswvmq\"\n \"ts\"=\"2026-03-09 10:20:20.677395\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-09 10:20:20.677659\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-09 10:20:20.677981\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n \"ts\"=\"2026-03-09 10:30:05.795084\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n \"ts\"=\"2026-03-09 10:30:05.795256\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"node-limit-test-rmm4pp\"\n \"ts\"=\"2026-03-09 10:31:17.926615\" \"level\"=0 \"msg\"=\"finished deleting created resources\"\n • [1230.077 seconds]\n ------------------------------\n\n Ran 1 of 1 Specs in 1230.077 seconds\n SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped\nCommand Error: \u003cnil\u003e\nDeserializaion Error: invalid character 'i' in literal true (expecting 'r')","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f","failure_pattern":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')","search_query":"Deserializaion Error: invalid character 'i' in literal true (expecting 'r')"}}],"failure_pattern_summary":{"signature_count":20,"failure_pattern_ids":["6438bade2f888700e3bb655208e715879ccf9a9cdbe3f13993b77fd7e78b922f"],"matched_occurrences":20,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030936664423010304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4364,"pr_state":"open","pr_sha":"2e6c30457cd0d9389f2bee67f73db6e237bed604","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-09T09:20:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030932636209254400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4363,"pr_state":"closed","pr_sha":"f758798f5312646231a1a1b167f59b4e0568ad51","final_merged_sha":"523a271a3a332b29ad83ffe992284dea47e7a593","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T09:04:07Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"436834d2802430fecac506bcccc57990bd7b7d1870474f9236eea4878c670e33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030932636209254400","occurred_at":"2026-03-09T09:04:07Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-09T09:33:46.213Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-09T09:33:48.541Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-09T09:33:49.543Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=3ca8d38bbb29123457df59c9bb94bbac5baa3103c704b582a1758d093cbef96d\ntime=2026-03-09T09:37:20.479Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9254400/providers/Microsoft.Resources/deployments/3ca8d38bbb29123457df59c9bb94bbac5baa3103c704b582a1758d093cbef96d/operationStatuses/08584285580565790414\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9254400/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9254400/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j9254400\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 2954c5b2-21d3-4d00-93b2-68482b979c33:3/9/2026 9:35:03 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4362/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030927466758934528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4362,"pr_state":"closed","pr_sha":"a72618f2244da3fc7983c01e47c67c3a5db80ff9","final_merged_sha":"a72618f2244da3fc7983c01e47c67c3a5db80ff9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T08:43:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4357/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030923762999758848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4357,"pr_state":"closed","pr_sha":"8826d13e4f5f9774116b844ebc3e29ad28f8e3f4","final_merged_sha":"8826d13e4f5f9774116b844ebc3e29ad28f8e3f4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T08:28:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4351/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030922443656269824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4351,"pr_state":"closed","pr_sha":"100f45d7deb453acb48c998c81b2a115cc90dce0","final_merged_sha":"131806c6037c274a80c20df0a02233247a78c57e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T08:23:37Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"65fbb10edeb3255663df2ddf0751c08e9df71aed49b3d02a024b23f1402b46fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4351/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030922443656269824","occurred_at":"2026-03-09T08:23:37Z","failed_at":"e2e","signature_id":"41fe23729393ed6e16ba75bf2c5f42eccc16cce18aafa85eb4f4bcc1fedfea58","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:229]: Unexpected error:\n \u003c*errors.joinError | 0xc0010fa300\u003e: \n cluster listing in non-existent resource group: expected error but none occurred\n node pool listing in RG with no clusters: expected error containing keywords 'parent resource not found', got: GET https://rp.j6269824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-97522w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n node pool listing in non-existent resource group: expected error containing keywords 'resource group not found', got: GET https://rp.j6269824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/non-existent-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*errors.errorString | 0xc000da9680\u003e{\n s: \"cluster listing in non-existent resource group: expected error but none occurred\",\n },\n \u003c*errors.errorString | 0xc001034ca0\u003e{\n s: \"node pool listing in RG with no clusters: expected error containing keywords 'parent resource not found', got: GET https://rp.j6269824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-97522w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n },\n \u003c*errors.errorString | 0xc000e6b220\u003e{\n s: \"node pool listing in non-existent resource group: expected error containing keywords 'resource group not found', got: GET https://rp.j6269824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/non-existent-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:229]: Unexpected error:\n \u003c*errors.joinError | 0xc0010fa300\u003e: \n cluster listing in non-existent resource group: expected error but none occurred\n node pool listing in RG with no clusters: expected error containing keywords 'parent resource not found', got: GET https://rp.j6269824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-97522w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n node pool listing in non-existent resource group: expected error containing keywords 'resource group not found', got: GET https://rp.j6269824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/non-existent-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*errors.errorString | 0xc000da9680\u003e{\n s: \"cluster listing in non-existent resource group: expected error but none occurred\",\n },\n \u003c*errors.errorString | 0xc001034ca0\u003e{\n s: \"node pool listing in RG with no clusters: expected error containing keywords 'parent resource not found', got: GET https://rp.j6269824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-97522w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n },\n \u003c*errors.errorString | 0xc000e6b220\u003e{\n s: \"node pool listing in non-existent resource group: expected error containing keywords 'resource group not found', got: GET https://rp.j6269824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/non-existent-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030915878593236992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4188,"pr_state":"closed","pr_sha":"af664ab2cfddb3f00ea65a9a5de003735aef037b","final_merged_sha":"ad7003e5a941946bb5a1fed27c539b6ee3d95872","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-09T07:57:32Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030904647304089600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4361,"pr_state":"closed","pr_sha":"e0a41cb5a28eae60e4edaaf959b07090458f697c","final_merged_sha":"5b9af084c4315b645526849e9e40d9cae69e886f","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-09T07:12:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4357/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030894978460291072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4357,"pr_state":"closed","pr_sha":"8826d13e4f5f9774116b844ebc3e29ad28f8e3f4","final_merged_sha":"8826d13e4f5f9774116b844ebc3e29ad28f8e3f4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T06:34:29Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4360/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030873068712235008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4360,"pr_state":"closed","pr_sha":"55a8598497b90edceae94bdec3d7e8651b9b5067","final_merged_sha":"55a8598497b90edceae94bdec3d7e8651b9b5067","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T05:07:25Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030842616056123392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4359,"pr_state":"closed","pr_sha":"249dc9319634a01662b1576f9bece1c87ba3de28","final_merged_sha":"249dc9319634a01662b1576f9bece1c87ba3de28","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-09T03:06:25Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4357/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030801720379969536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4357,"pr_state":"closed","pr_sha":"56807a7601ee3a7fcc240ff4eb2427ad546c3928","final_merged_sha":"8826d13e4f5f9774116b844ebc3e29ad28f8e3f4","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-09T00:23:55Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"dbdb9ee6b3538778da87415dee618ea36a8aa31824ca2b95632dc9e70f9eb488","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4357/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2030801720379969536","occurred_at":"2026-03-09T00:23:55Z","failed_at":"provision","signature_id":"1e3c66f1b3aaa8e9531a0447e5fe758d4cfa54cfe1d96c0e23b03169e36088b8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"step graph","failure_text":"time=2026-03-09T01:19:23.519Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: Shell\\n Command: ./istio.sh\\n\"\ntime=2026-03-09T01:19:40.786Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"error running Shell Step, failed to execute shell command: + set -euo pipefail\\n+ ISTIO_NAMESPACE=aks-istio-system\\n+ echo '********** Check istio is up and running **************'\\n********** Check istio is up and running **************\\n++ kubectl get pods -n aks-istio-system -l istio.io/rev=asm-1-26 --field-selector=status.phase=Running --no-headers\\n++ wc -l\\n+ ISTIO_PODS_COUNT=2\\n+ [[ 2 -lt 2 ]]\\n+ echo '********** Download istioctl **************'\\n********** Download istioctl **************\\n++ uname\\n+ OS=Linux\\n+ '[' Linux = Darwin ']'\\n+ OSEXT=linux\\n++ uname -m\\n+ LOCAL_ARCH=x86_64\\n+ case \\\"${LOCAL_ARCH}\\\" in\\n+ ISTIO_ARCH=amd64\\n+ ISTIO_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz\\n+ SHA256_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz -o istio-1.28.3-linux-amd64.tar.gz\\n exit status 6\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"00bf42e30ff9d333f122e52fc30a4967eab6684f10cfa8840e0bee5874b06083","failure_pattern":"error running Shell Step, failed to execute shell command: + set -euo pipefail + ISTIO_NAMESPACE=aks-istio-system + echo '********** Check istio is up and running **************' ********** Check istio is up and running ************** ++ kubectl get pod","search_query":"error running Shell Step, failed to execute shell command: + set -euo pipefail\\n+ ISTIO_NAMESPACE=aks-istio-system\\n+ echo '********** Check istio is up and running **************'\\n********** Check istio is up and running **************\\n++ kubectl get pods -n aks-istio-system -l istio.io/rev=asm-1-26 --field-selector=status.phase=Running --no-headers\\n++ wc -l\\n+ ISTIO_PODS_COUNT=2\\n+ [[ 2 -lt 2 ]]\\n+ echo '********** Download istioctl **************'\\n********** Download istioctl **************\\n++ uname\\n+ OS=Linux\\n+ '[' Linux = Darwin ']'\\n+ OSEXT=linux\\n++ uname -m\\n+ LOCAL_ARCH=x86_64\\n+ case \\\"${LOCAL_ARCH}\\\" in\\n+ ISTIO_ARCH=amd64\\n+ ISTIO_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz\\n+ SHA256_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz -o istio-1.28.3-linux-amd64.tar.gz\\n exit status 6"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["00bf42e30ff9d333f122e52fc30a4967eab6684f10cfa8840e0bee5874b06083"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}