{"meta":{"date":"2026-03-23","timezone":"UTC","generated_at":"2026-10-31T23:15:15Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":68,"failed_runs":46,"runs_with_occurrences":46,"runs_with_failure_pattern_matches":35,"runs_with_unmatched_occurrences":11,"runs_without_occurrence_rows":22,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036229627118096384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4589,"pr_state":"closed","pr_sha":"436ddbf1525e9ebf05d5362c884575361b0518ae","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T23:52:29Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b5c697900d1c6543e16fde5e3f3f06d751341b598c1ca755001feba942df7f6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036229627118096384","occurred_at":"2026-03-23T23:52:29Z","failed_at":"provision","signature_id":"eb42f536566c06969e79bb4dd530b90fade97f988fe3d18503444c2160414c7b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/svc-mgmt-permissions","test_suite":"step graph","failure_text":"time=2026-03-24T00:31:24.809Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions description=\"Step svc-mgmt-permissions\\n  Kind: ARM\\n  Template: templates/svc-mgmt-permissions.bicep\\n  Parameters: configurations/svc-mgmt-permissions.tmpl.bicepparam\"\ntime=2026-03-24T00:31:25.578Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions\ntime=2026-03-24T00:31:26.601Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions deployment=7a15931e1f96af913f2cb12fb2d11d6e14f7d5aaaf1b1b54309a877efbe4f835\ntime=2026-03-24T00:31:56.909Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8096384-mgmt-1/providers/Microsoft.Resources/deployments/7a15931e1f96af913f2cb12fb2d11d6e14f7d5aaaf1b1b54309a877efbe4f835/operationStatuses/08584272945994994062\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentExists\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The role assignment already exists. The ID of the existing role assignment is c2458f5d40a058a9b4d76c1a122ba64b.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2c08b2b0f48b1745f538e5727c28908ad9a04645767c115dc1cc38892e968435","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentExists; detail message The role assignment already exists. The ID of the existing role assignment is \u003chex\u003e.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2c08b2b0f48b1745f538e5727c28908ad9a04645767c115dc1cc38892e968435"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036226415044595712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4586,"pr_state":"closed","pr_sha":"0d693572f01e0ca9ac068302880fa6634c11b39b","final_merged_sha":"89f77ed8135cc386d33aa903c6bdcc1e930b7268","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T23:39:43Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c89c3657fdf453692ebbd6ec77428a5e1f14e5776a44d967714ddaadb2a97fcb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036226415044595712","occurred_at":"2026-03-23T23:39:43Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-23T23:55:28.903Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-23T23:55:29.544Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:01038e7de14b78d702d2849c3aad72fd25903c4765af63cf16aa3398f5d5f2dd]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b7c83e5834c5e97ef17af96ddbcba9ed7062371a1db1a46171ff7ae1ae137aec] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:b7bd3a6bf3160b82af89c345c58b1fd3a936801c8c65fef7ed2111dc71c766a3]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:bef5c287-bf4c-42fe-8749-19e46529efe9]] prometheusSpec:map[externalLabels:map[cluster:prow-j4595712-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j4595712-mgmt-1-lfli.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-f441ffb297654dd29a83464cc6a20f65/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:eb5dcb5c4f6e229d6dfe17b492291e06bea7ba19f080d48a7afc663e4420386c] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j4595712-mgmt-1-lfli.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-3c038c95ecfb4de8b8a9349e9256b31f/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-23T23:55:29.544Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T23:55:29.855Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus namespace=prometheus\ntime=2026-03-23T23:55:30.215Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus dryRun=false\ntime=2026-03-23T23:55:30.215Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T23:55:30.392Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T23:55:56.499Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed pre-install: Unauthorized\"\ntime=2026-03-23T23:55:56.499Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T23:55:56.499Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T23:56:03.840Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus release=arohcp-monitor namespace=prometheus status=failed description=\"Release \\\"arohcp-monitor\\\" failed: failed pre-install: Unauthorized\" values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:01038e7de14b78d702d2849c3aad72fd25903c4765af63cf16aa3398f5d5f2dd]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b7c83e5834c5e97ef17af96ddbcba9ed7062371a1db1a46171ff7ae1ae137aec] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:b7bd3a6bf3160b82af89c345c58b1fd3a936801c8c65fef7ed2111dc71c766a3]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:bef5c287-bf4c-42fe-8749-19e46529efe9]] prometheusSpec:map[externalLabels:map[cluster:prow-j4595712-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j4595712-mgmt-1-lfli.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-f441ffb297654dd29a83464cc6a20f65/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:eb5dcb5c4f6e229d6dfe17b492291e06bea7ba19f080d48a7afc663e4420386c] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j4595712-mgmt-1-lfli.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-3c038c95ecfb4de8b8a9349e9256b31f/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-23T23:56:03.840Z level=INFO msg=\"No resource info available in release, skipping resource diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus infoNil=false resourceCount=0\ntime=2026-03-23T23:56:03.840Z level=INFO msg=\"Kusto deep link for namespace-level troubleshooting\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus kustoLinkNamespace=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/3yOvaqDQBSEe5/isJWCiuyiheAb3PJWNpd1HdRL9ofdEyTgwwdTmFTpho8Z5jPesd4c4o9fUnbQviKCeLNIrG2gCbwDjvJZM06cC9nIrmpUJdWvVH3b9qoZRUF1TV86bTeKorgOnLZIQRv8nYmGgUSI3oJX3JPIDgrR/8Pw26Skm19KCn5+TbKDfJwRaXpcjHQy5Ye7TuYZAAD//wnmPLTfAAAA\"\ntime=2026-03-23T23:56:03.840Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T23:56:03.841Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed pre-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9e8faf9dd151e5316337afa8f0d694f70b9e3597eb997a02e9a233307595638","failure_pattern":"failed pre-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed pre-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d9e8faf9dd151e5316337afa8f0d694f70b9e3597eb997a02e9a233307595638"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036223400287932416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4590,"pr_state":"closed","pr_sha":"bfdd6afc43a8add4e957334ea4372e6b4768bf60","final_merged_sha":"bfdd6afc43a8add4e957334ea4372e6b4768bf60","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T23:27:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036217151928406016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4586,"pr_state":"closed","pr_sha":"0d693572f01e0ca9ac068302880fa6634c11b39b","final_merged_sha":"89f77ed8135cc386d33aa903c6bdcc1e930b7268","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T23:02:54Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9e836f102e5b2659b70fa4fb16b9a3972a6d41cdbf427645e894ab3b9195069b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036217151928406016","occurred_at":"2026-03-23T23:02:54Z","failed_at":"provision","signature_id":"55f959212b276953d69124b2535dda8bcd5c609c88c398d3ca208722abf25ab9","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy","test_suite":"step graph","failure_text":"time=2026-03-23T23:25:49.144Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-23T23:25:49.714Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5 imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:490c3b795cb12bd9e5a3680bb7dabd7ea02799c30b14ed960611517c5d808e6b imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[cpu:50m memory:100Mi] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-03-23T23:25:49.714Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy\ntime=2026-03-23T23:25:49.961Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy namespace=secret-sync-controller\ntime=2026-03-23T23:25:50.287Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy dryRun=false\ntime=2026-03-23T23:25:50.287Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy\ntime=2026-03-23T23:25:50.435Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy\ntime=2026-03-23T23:25:59.155Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy err=\"Unauthorized\\nUnauthorized\\nUnauthorized\\nUnauthorized\"\ntime=2026-03-23T23:25:59.155Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy\ntime=2026-03-23T23:25:59.155Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy\ntime=2026-03-23T23:26:04.109Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy release=secret-sync-controller namespace=secret-sync-controller status=pending-install description=\"Initial install underway\" values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5 imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:490c3b795cb12bd9e5a3680bb7dabd7ea02799c30b14ed960611517c5d808e6b imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[cpu:50m memory:100Mi] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-03-23T23:26:04.110Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/7SVT4vbPBDG7/4Ug08bsJaQsC+8KXvYlp4KpbSlhy09yNKDVxt5xkhKQiAfvjjOH7NN0hTik2A08/xG6JEm80gUEGURDCI9ktVJJ1163En5CpM+ObazmILjqqAu9FnXOISwBG8jsdFmHx79zIiI8g9+ERPCV/HIC8ojTEBScc1GaVs7boN5cSF3mwrrkgQVdjsnCt47to6rXp2KSQI6lBFOQbxHULVmXaHTKo9Fe8k38DMiHtoiKHiY5IQPjfWPd0zfa//Q3lmdHFdPtnYxOuEv4p1Z/x1oAnSCWjS2XZpD1c2lk8zByoLXt6VYeNy6813L5gVmrsTbzpu3ZSy7UijNLElvb3sYgNcl/EDa3a3+g/bVz+mUMdWJd3Vr0h8+HQa6o8mKEeIZpw1DPm+8gXl9Hw6M6tvyFOobwtIZPBkjC05X/+0XfuLs17tsvigRGAnxYzu5Yrah1QsCKLkaMem6oRJpBTDdtW224bt8Mp78p8ZTNZl+n0xnk4fZw/g5H9H9PV3K+f85H42yDb2KY5o7to+OGaE3b4XpOGb78/XtYM021ARpd4+NFtSI7ZKvEynIS1VQjRh11UpKsAhUrnuHt4jmdwAAAP//lzNCfhUIAAA=\"\ntime=2026-03-23T23:26:04.110Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy\ntime=2026-03-23T23:26:04.110Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: Unauthorized\\nUnauthorized\\nUnauthorized\\nUnauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb12dfd3a394833bbc6aede7c12be18602b7c7939b9dbea2119c93e9d9d2d2","failure_pattern":"Unauthorized Unauthorized Unauthorized Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: Unauthorized\\nUnauthorized\\nUnauthorized\\nUnauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fafb12dfd3a394833bbc6aede7c12be18602b7c7939b9dbea2119c93e9d9d2d2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036214835779211264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4589,"pr_state":"closed","pr_sha":"8bb1da83f688f7cb641d2909085cbe6491187957","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T22:53:38Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"54986cda860235b99f90400e21257da52146441c079028eca3f1fab5cc811039","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036214835779211264","occurred_at":"2026-03-23T22:53:38Z","failed_at":"provision","signature_id":"eb42f536566c06969e79bb4dd530b90fade97f988fe3d18503444c2160414c7b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/svc-mgmt-permissions","test_suite":"step graph","failure_text":"time=2026-03-23T23:34:29.701Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions description=\"Step svc-mgmt-permissions\\n  Kind: ARM\\n  Template: templates/svc-mgmt-permissions.bicep\\n  Parameters: configurations/svc-mgmt-permissions.tmpl.bicepparam\"\ntime=2026-03-23T23:34:30.234Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions\ntime=2026-03-23T23:34:31.142Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions deployment=1274b489e405478a98e50259476f20a0959f9f26bb68b51343591eb230be2a5e\ntime=2026-03-23T23:35:01.387Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9211264-mgmt-1/providers/Microsoft.Resources/deployments/1274b489e405478a98e50259476f20a0959f9f26bb68b51343591eb230be2a5e/operationStatuses/08584272980148477728\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentExists\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The role assignment already exists. The ID of the existing role assignment is 37ef272e2ef651f1be383cfc74698550.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2c08b2b0f48b1745f538e5727c28908ad9a04645767c115dc1cc38892e968435","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentExists; detail message The role assignment already exists. The ID of the existing role assignment is \u003chex\u003e.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2c08b2b0f48b1745f538e5727c28908ad9a04645767c115dc1cc38892e968435"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036210246233362432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4513,"pr_state":"open","pr_sha":"e8857103e40f92eaadf121aad197ac99902486c4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T22:35:28Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e688c2dd7a5ea6f89f16c43fb4afa1a316888b45a033a8ba9e28f9f76e744e31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036210246233362432","occurred_at":"2026-03-23T22:35:28Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036208145574924288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4586,"pr_state":"closed","pr_sha":"0d693572f01e0ca9ac068302880fa6634c11b39b","final_merged_sha":"89f77ed8135cc386d33aa903c6bdcc1e930b7268","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T22:27:07Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1be7e1fa4e8810bc7e8eac5fa96376732f996f473ad850c6ee144bcd074a5eac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036208145574924288","occurred_at":"2026-03-23T22:27:07Z","failed_at":"provision","signature_id":"af99b9283742f286e546fac9212a5301d331519de67f18f8db49375b0ce9e2c0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-policies","test_suite":"step graph","failure_text":"time=2026-03-23T22:51:44.579Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies description=\"Step deploy-policies\\n Kind: Helm\\n\"\ntime=2026-03-23T22:51:45.140Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies values=map[]\ntime=2026-03-23T22:51:45.140Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-23T22:51:45.384Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies namespace=open-cluster-management-policies\ntime=2026-03-23T22:51:45.704Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies dryRun=false\ntime=2026-03-23T22:51:45.704Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-23T22:51:45.832Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-23T22:51:49.330Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies err=\"create: failed to create: Unauthorized\"\ntime=2026-03-23T22:51:49.330Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-23T22:51:49.330Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-23T22:51:49.566Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies err=\"failed to get status for release policy: release: not found\"\ntime=2026-03-23T22:51:49.566Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-23T22:51:49.566Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies err=\"error running Helm release deployment Step, failed to deploy helm release: create: failed to create: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82051b3fe3ade36d94e25e5df07830f73170ae636dad9efb1d1d600505f66d35","failure_pattern":"create: failed to create: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: create: failed to create: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82051b3fe3ade36d94e25e5df07830f73170ae636dad9efb1d1d600505f66d35"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036207653574676480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4584,"pr_state":"closed","pr_sha":"4999eef2bb460e178d184cded32a17edd3e2ffea","final_merged_sha":"4999eef2bb460e178d184cded32a17edd3e2ffea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T22:25:10Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036202683802587136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4482,"pr_state":"closed","pr_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","final_merged_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T22:05:25Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036201236570574848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4589,"pr_state":"closed","pr_sha":"0ddc9ab928704073de9a0a1a74f58e0b3c103276","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T21:59:40Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e0fa23bcba7ca785efb1fb2a226fd43a1cfbecc77f7b6897e5c09c84b037201a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036201236570574848","occurred_at":"2026-03-23T21:59:40Z","failed_at":"provision","signature_id":"eb42f536566c06969e79bb4dd530b90fade97f988fe3d18503444c2160414c7b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/svc-mgmt-permissions","test_suite":"step graph","failure_text":"time=2026-03-23T22:35:15.970Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions description=\"Step svc-mgmt-permissions\\n  Kind: ARM\\n  Template: templates/svc-mgmt-permissions.bicep\\n  Parameters: configurations/svc-mgmt-permissions.tmpl.bicepparam\"\ntime=2026-03-23T22:35:16.418Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions\ntime=2026-03-23T22:35:17.444Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions deployment=ee5c573b597439779ce725913807e1ec592a2cdb091cfb47e69e84f48758905a\ntime=2026-03-23T22:35:47.639Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0574848-mgmt-1/providers/Microsoft.Resources/deployments/ee5c573b597439779ce725913807e1ec592a2cdb091cfb47e69e84f48758905a/operationStatuses/08584273015687315026\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentExists\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The role assignment already exists. The ID of the existing role assignment is ffb777f2900b52a28855ed2d9ce453f6.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2c08b2b0f48b1745f538e5727c28908ad9a04645767c115dc1cc38892e968435","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentExists; detail message The role assignment already exists. The ID of the existing role assignment is \u003chex\u003e.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2c08b2b0f48b1745f538e5727c28908ad9a04645767c115dc1cc38892e968435"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036193819497598976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4585,"pr_state":"closed","pr_sha":"5ed7b324786bb326a526a06fec3e2fdc2ceec01f","final_merged_sha":"5ed7b324786bb326a526a06fec3e2fdc2ceec01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-23T21:30:11Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4166647e59b666c25cc9faf4fdf0e0694c027f2ccbdf0db534edc3012a8eb0b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036193819497598976","occurred_at":"2026-03-23T21:30:11Z","failed_at":"provision","signature_id":"5564627504dc0ffa8b2a1cfb61f4b200a8883ded65fa2c1a4914a1dd7bd7f88c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config","test_suite":"step graph","failure_text":"time=2026-03-23T22:08:31.203Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-23T22:08:31.798Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5d1fc3770795996bc22512c28adf4623ecb8a69eec3018a88e34a87bcba3366a global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-23T22:08:31.798Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-23T22:08:31.871Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config namespace=multicluster-engine\ntime=2026-03-23T22:08:32.020Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config dryRun=false\ntime=2026-03-23T22:08:32.020Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-23T22:08:32.160Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-23T22:08:59.219Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed post-install: warning: Hook post-install multicluster-engine-local-cluster/templates/finalize.job.yaml failed: Unauthorized\"\ntime=2026-03-23T22:08:59.219Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-23T22:08:59.219Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-23T22:09:03.131Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config release=mce-config namespace=multicluster-engine status=failed description=\"Release \\\"mce-config\\\" failed: failed post-install: warning: Hook post-install multicluster-engine-local-cluster/templates/finalize.job.yaml failed: Unauthorized\" values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5d1fc3770795996bc22512c28adf4623ecb8a69eec3018a88e34a87bcba3366a global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-23T22:09:03.132Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yUT2/aTBDG7/4UI59AwhEyel+1VDnQtKcordT0lKqH9e5gNqxnrNkFhMSHr/wPUxICpb0+nvnpeXbGEzkMIOh5JRo93IJRQQWVORxw9ow63FsyUx/EUj6CRvqiCtxLuEaqFV8q3cnDHxEAQHzvVj6gOAwzY5jumOY2j0cQO9bKJbr5/IrQtD8oUjmau0Z9xFBVLrYlil/YeUiK+nuBFLpOX1V07Y8oa6txpjWvqO5d7v0kqjKUaKYg7BxKsk6rimLlgm1hCVJuCU/jctFJyc7qbYvz6gyjjfKNHX60ZCzVr8ElUpfgINT0vN3ruLnoaSsLO8z6jo53BKqCvgh71PpG6hO0ywEzY77SJywdb6sA/R41ThbsA5rqdVr5/AQe9q9Rs2tbvEYhRRq7nHNRBW5YlodP0+90byN3nCl3YiD/fMJ/MNorCU3+1tHrtA5zcjUu2InjZTjT0i/AX//Lv6Ne+O9R13JK4VLlKvBJQvTzQ7RcZSiEAf3n6oz6aAebBQpCsAX6oIoSMgwbRIKBUQEreRCn4/T/ZDxJ0sn3NJ2O300n6VM8hJsbeKPmv/dP8XAY7eCZLcHSkrm1RCgHx58J+pt/eOyPr3y0g1K4+tobHUHJpim+DDICx/kICvRe5RWSxaBAtj0Ib9DrXwEAAP//r5hklaIGAAA=\"\ntime=2026-03-23T22:09:03.133Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-23T22:09:03.134Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: warning: Hook post-install multicluster-engine-local-cluster/templates/finalize.job.yaml failed: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc8beeb6a980951a0fb786a044acdf2a97f2dc59d3c7cd4270302ca9096fa930","failure_pattern":"failed post-install: warning: Hook post-install multicluster-engine-local-cluster/templates/finalize.job.yaml failed: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: warning: Hook post-install multicluster-engine-local-cluster/templates/finalize.job.yaml failed: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bc8beeb6a980951a0fb786a044acdf2a97f2dc59d3c7cd4270302ca9096fa930"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036191267292975104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4556,"pr_state":"open","pr_sha":"047f4394fbfb3ee76f54b58c25ba5019edf9b456","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T21:20:03Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9f6c2e5b54f8378f49438b9553c68c4f59e8db7cb6d2503fbdcc8d16c797f1a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036191267292975104","occurred_at":"2026-03-23T21:20:03Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-23T21:43:19.090Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n  Kind: Shell\\n  Command: make deploy\\n\"\ntime=2026-03-23T21:43:37.165Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=a4d3a2a1-c8bb-4792-8cae-355b5030e10c \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=bf8ce325-899b-4289-8318-606037f6dfde \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj2975104.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj2975104.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j2975104 -n arohcp-prow-dbcs-j2975104 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2975104/providers/Microsoft.Network/dnszones/j2975104.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j2975104-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j2975104-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n  -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n  --set serviceAccountName=clusters-service \\\\\\n  --set environment=arohcpprow \\\\\\n  --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n  --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n  --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n  --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n  --set tlsCertificatesEnabled=true \\\\\\n  --set denyAssignments=disabled \\\\\\n  --set tenantId=${TENANT_ID} \\\\\\n  --set region=westus3 \\\\\\n  --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n  --set replicas=3 \\\\\\n  --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n  --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n  --set imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set global.imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n  --set batchProcessesDryRun=true \\\\\\n  --set batchProcesses=\\\"\\\" \\\\\\n  --set fpaCertName=firstPartyCert2 \\\\\\n  --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n  --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n  --set shard.consumerName=\\\"hcp-underlay-j2975104-mgmt-1\\\" \\\\\\n  --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n  --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultMiClientId=\\\"51a31a28-8560-4ccd-b603-6213d8bf44ac\\\" \\\\\\n  --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n  --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n  --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j2975104-mgmt-1'))\\\") \\\\\\n  --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2975104-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j2975104-mgmt-1 \\\\\\n  --set databaseHost=${DB_HOST} \\\\\\n  --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n  --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n  --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n  --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n  --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n  --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n  --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n  --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n  --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n  --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n  --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n  --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n  --set tracing.address=http://ingest.observability:4318 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n  --set provisionShardClusterLimit=100 \\\\\\n  --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n  --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n  --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n  --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n  --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n  --set azureOperatorsMI.roleSetName=dev \\\\\\n  --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: create: failed to create: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036191226046189568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4586,"pr_state":"closed","pr_sha":"0d693572f01e0ca9ac068302880fa6634c11b39b","final_merged_sha":"89f77ed8135cc386d33aa903c6bdcc1e930b7268","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T21:19:53Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2730422fb90196537f280740f5fa603c67163d984101c91604324ccd14918977","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036191226046189568","occurred_at":"2026-03-23T21:19:53Z","failed_at":"provision","signature_id":"7a7d0f5d0e8ab95b7b5e95c3c532a9322e5dd52c30984711b49e7a1f1a32e9d3","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/management/output","test_suite":"step graph","failure_text":"time=2026-03-23T21:42:27.275Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=management step=output description=\"Step output\\n  Kind: ARM\\n  Template: ../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-03-23T21:42:27.852Z level=DEBUG+2 msg=\"Waiting for existing deployment to complete\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=management step=output deployment=14278ec7016f40fafde1317e05d76b8961b538d2c814ee2123a7c263f16c5d3a\ntime=2026-03-23T21:45:29.019Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=management step=output err=\"failed to run ARM step: failed to poll previously-existing deployment: timed out waiting for pre-existing deployment to finish\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a3901d9621d6c16ce658f06646595e4a1cded970925c8d80d869b6f2b3b95e95","failure_pattern":"failed to run ARM step: failed to poll previously-existing deployment: timed out waiting for pre-existing deployment to finish","search_query":"failed to run ARM step: failed to poll previously-existing deployment: timed out waiting for pre-existing deployment to finish"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a3901d9621d6c16ce658f06646595e4a1cded970925c8d80d869b6f2b3b95e95"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036182173475672064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4589,"pr_state":"closed","pr_sha":"f0a875f10b3344fa2d695fd18992449d792c8085","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T20:43:29Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8403b77435d235af07e548b8689d844c89d477790bc1fb0a84c7e458ccbe5b7d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036182173475672064","occurred_at":"2026-03-23T20:43:29Z","failed_at":"provision","signature_id":"eb42f536566c06969e79bb4dd530b90fade97f988fe3d18503444c2160414c7b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/svc-mgmt-permissions","test_suite":"step graph","failure_text":"time=2026-03-23T21:20:54.796Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions description=\"Step svc-mgmt-permissions\\n  Kind: ARM\\n  Template: templates/svc-mgmt-permissions.bicep\\n  Parameters: configurations/svc-mgmt-permissions.tmpl.bicepparam\"\ntime=2026-03-23T21:20:55.446Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions\ntime=2026-03-23T21:20:56.735Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions deployment=18cad54200e7607a0caa73878ae72d675e10111a6f936a591fd9daeaa39784c0\ntime=2026-03-23T21:21:26.980Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5672064-mgmt-1/providers/Microsoft.Resources/deployments/18cad54200e7607a0caa73878ae72d675e10111a6f936a591fd9daeaa39784c0/operationStatuses/08584273060294549784\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentExists\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The role assignment already exists. The ID of the existing role assignment is 7f32bc67708b533c99857e6f580b6d47.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2c08b2b0f48b1745f538e5727c28908ad9a04645767c115dc1cc38892e968435","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentExists; detail message The role assignment already exists. The ID of the existing role assignment is \u003chex\u003e.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2c08b2b0f48b1745f538e5727c28908ad9a04645767c115dc1cc38892e968435"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036181355787718656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4554,"pr_state":"closed","pr_sha":"1ba8557e7aa3030dfdb65d2dedde1be704afe79f","final_merged_sha":"cab9df44f4aaadfa6301e36bed1fb8403e84a43b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T20:40:39Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"69aaf7bf7750c6f8cdc37eddc6e1f4a296cbe5bd61eb662d89f057216b285ae4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036181355787718656","occurred_at":"2026-03-23T20:40:39Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036178657696813056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4586,"pr_state":"closed","pr_sha":"0d693572f01e0ca9ac068302880fa6634c11b39b","final_merged_sha":"89f77ed8135cc386d33aa903c6bdcc1e930b7268","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T20:29:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d8824ea64cdf6c62fc47f3efc634753bcc19d9ce510509f7bb08b6123daee24b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036178657696813056","occurred_at":"2026-03-23T20:29:56Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-23T21:07:24.361Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n  Kind: Shell\\n  Command: make deploy\\n\"\ntime=2026-03-23T21:08:34.422Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=55915c73-d067-49b5-a2bf-3413afbb0134 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=4ffbf161-8d3e-4a13-9066-782cc4d019d9 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj6813056.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj6813056.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j6813056 -n arohcp-prow-dbcs-j6813056 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6813056/providers/Microsoft.Network/dnszones/j6813056.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j6813056-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j6813056-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n  -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n  --set serviceAccountName=clusters-service \\\\\\n  --set environment=arohcpprow \\\\\\n  --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n  --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n  --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n  --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n  --set tlsCertificatesEnabled=true \\\\\\n  --set denyAssignments=disabled \\\\\\n  --set tenantId=${TENANT_ID} \\\\\\n  --set region=westus3 \\\\\\n  --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n  --set replicas=3 \\\\\\n  --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n  --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n  --set imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set global.imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n  --set batchProcessesDryRun=true \\\\\\n  --set batchProcesses=\\\"\\\" \\\\\\n  --set fpaCertName=firstPartyCert2 \\\\\\n  --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n  --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n  --set shard.consumerName=\\\"hcp-underlay-j6813056-mgmt-1\\\" \\\\\\n  --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n  --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultMiClientId=\\\"6823beeb-33ae-4516-aab1-7798c8f63116\\\" \\\\\\n  --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n  --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n  --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j6813056-mgmt-1'))\\\") \\\\\\n  --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6813056-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j6813056-mgmt-1 \\\\\\n  --set databaseHost=${DB_HOST} \\\\\\n  --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n  --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n  --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n  --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n  --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n  --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n  --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n  --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n  --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n  --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n  --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n  --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n  --set tracing.address=http://ingest.observability:4318 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n  --set provisionShardClusterLimit=100 \\\\\\n  --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n  --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n  --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n  --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n  --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n  --set azureOperatorsMI.roleSetName=dev \\\\\\n  --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036178598661984256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4556,"pr_state":"open","pr_sha":"047f4394fbfb3ee76f54b58c25ba5019edf9b456","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T20:29:37Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3ef598878a4d06c370c4ce4c61dfa58aad4c5524749143df2b5da2695995bf1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036178598661984256","occurred_at":"2026-03-23T20:29:37Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036177881456971776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4482,"pr_state":"closed","pr_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","final_merged_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-23T20:26:51Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9c009c3351eefac5ffd37d05ba55e70161eaa579a2124357cb5bff33915cf0cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036177881456971776","occurred_at":"2026-03-23T20:26:51Z","failed_at":"provision","signature_id":"5564627504dc0ffa8b2a1cfb61f4b200a8883ded65fa2c1a4914a1dd7bd7f88c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config","test_suite":"step graph","failure_text":"time=2026-03-23T21:04:32.625Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-23T21:04:33.148Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5d1fc3770795996bc22512c28adf4623ecb8a69eec3018a88e34a87bcba3366a global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-23T21:04:33.148Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-23T21:04:33.214Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config namespace=multicluster-engine\ntime=2026-03-23T21:04:33.345Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config dryRun=false\ntime=2026-03-23T21:04:33.345Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-23T21:04:33.472Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-23T21:05:57.245Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69c1a715a03f4d0001c61afb.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69c1a715a03f4d0001c61afb.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)\"\ntime=2026-03-23T21:05:57.245Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-23T21:05:57.245Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-23T21:05:57.394Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed to get status for release mce-config: query: failed to query with labels: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/multicluster-engine/secrets?labelSelector=name%3Dmce-config%2Cowner%3Dhelm\\\\\\\": Post \\\\\\\"https://guard.69c1a715a03f4d0001c61afb.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69c1a715a03f4d0001c61afb.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (get secrets)\"\ntime=2026-03-23T21:05:57.394Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-23T21:05:57.394Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69c1a715a03f4d0001c61afb.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69c1a715a03f4d0001c61afb.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2a1f671a46ca05a4f95515404a4035dd108dc9eb264bd0b098221cb0d38c8032","failure_pattern":"failed post-install: an error on the server (\"Internal Server Error: \\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\": Post \\\\\"\u003curl\u003e\": proxy error from localhost:9443 while dialing","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69c1a715a03f4d0001c61afb.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69c1a715a03f4d0001c61afb.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2a1f671a46ca05a4f95515404a4035dd108dc9eb264bd0b098221cb0d38c8032"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036175828965920768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4588,"pr_state":"closed","pr_sha":"aedf0e0c2a3f6ead12b269ae356d756a76ec10a7","final_merged_sha":"582667ceb51a1eed2a4604a71a440bcf5821ebe2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T20:18:42Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"96db0fac9091f0f43bfcb993bc1e9ffc6f42765727cb95d08fa0e49b77a98d73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036175828965920768","occurred_at":"2026-03-23T20:18:42Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-23T20:53:47.684Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-23T20:53:48.939Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:01038e7de14b78d702d2849c3aad72fd25903c4765af63cf16aa3398f5d5f2dd]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b7c83e5834c5e97ef17af96ddbcba9ed7062371a1db1a46171ff7ae1ae137aec] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:b7bd3a6bf3160b82af89c345c58b1fd3a936801c8c65fef7ed2111dc71c766a3]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:7ec6de8f-1c00-4a8f-9c52-258e1fa09011]] prometheusSpec:map[externalLabels:map[cluster:prow-j5920768-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j5920768-mgmt-1-bvfq.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-80a0578c8edd4126b2767f29342b397e/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:eb5dcb5c4f6e229d6dfe17b492291e06bea7ba19f080d48a7afc663e4420386c] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j5920768-mgmt-1-bvfq.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-30c0409761ed4d3ca84fd154689b07a0/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-23T20:53:48.939Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T20:53:49.266Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus namespace=prometheus\ntime=2026-03-23T20:53:49.671Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus dryRun=false\ntime=2026-03-23T20:53:49.671Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T20:53:49.874Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T20:53:50.460Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"cluster reachability check failed: kubernetes cluster unreachable: the server has asked for the client to provide credentials\"\ntime=2026-03-23T20:53:50.460Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T20:53:50.460Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T20:53:50.679Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed to get status for release arohcp-monitor: release: not found\"\ntime=2026-03-23T20:53:50.679Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T20:53:50.679Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: cluster reachability check failed: kubernetes cluster unreachable: the server has asked for the client to provide credentials\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df1532ddcdc2d62f97589fa9c092641092a3064f3184196911681c8bfee0ae44","failure_pattern":"cluster reachability check failed: kubernetes cluster unreachable: the server has asked for the client to provide credentials","search_query":"error running Helm release deployment Step, failed to deploy helm release: cluster reachability check failed: kubernetes cluster unreachable: the server has asked for the client to provide credentials"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df1532ddcdc2d62f97589fa9c092641092a3064f3184196911681c8bfee0ae44"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036160628850167808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4587,"pr_state":"closed","pr_sha":"8c0883912b4d06dc00be6d05ddd7734a18c22356","final_merged_sha":"49f81d94ae91271fb6f2b5b819c67709937251ba","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T19:18:18Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7d1446b9bc689bc33c6c0bb70be0b03fb899e82f8cf640f20d323c81baeefc62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036160628850167808","occurred_at":"2026-03-23T19:18:18Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-23T19:52:45.211Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-23T19:52:45.790Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:01038e7de14b78d702d2849c3aad72fd25903c4765af63cf16aa3398f5d5f2dd]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b7c83e5834c5e97ef17af96ddbcba9ed7062371a1db1a46171ff7ae1ae137aec] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:b7bd3a6bf3160b82af89c345c58b1fd3a936801c8c65fef7ed2111dc71c766a3]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:e3ec7c4a-d67f-417f-a298-01edac1d5653]] prometheusSpec:map[externalLabels:map[cluster:prow-j0167808-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j0167808-mgmt-1-ppdq.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-a90f9d625b434c29813a131df9e03268/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:eb5dcb5c4f6e229d6dfe17b492291e06bea7ba19f080d48a7afc663e4420386c] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j0167808-mgmt-1-ppdq.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-368fc8df388b4b90a814030d03dd4f3e/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-23T19:52:45.790Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T19:52:46.286Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace prometheus: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ecf60ba8bdd57b40e995d27f45ca5a2fb0a8082c735c55d75d0af026c1021bb","failure_pattern":"failed to apply namespace prometheus: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace prometheus: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8ecf60ba8bdd57b40e995d27f45ca5a2fb0a8082c735c55d75d0af026c1021bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036150236191133696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4586,"pr_state":"closed","pr_sha":"0d693572f01e0ca9ac068302880fa6634c11b39b","final_merged_sha":"89f77ed8135cc386d33aa903c6bdcc1e930b7268","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T18:33:19Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3b7665cbdf9399759000da1de9b5e18eed9df48d04688fa2676f8987fd206f65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036150236191133696","occurred_at":"2026-03-23T18:33:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036149543808012288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"closed","pr_sha":"7cddbd3cbc43f559f3df12084ec577bed405c99f","final_merged_sha":"4fe7725f48a2f3f1c0ac843b4434f502807b38e5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T18:24:45Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"afa4c8722a69a82101ed13aaad028bbf26e8ca12fe7dfeba9177c1bb95c1d32e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036149543808012288","occurred_at":"2026-03-23T18:24:45Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-23T19:20:44.752Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n  Kind: Shell\\n  Command: make deploy\\n\"\ntime=2026-03-23T19:22:36.961Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=9fc8d3f9-d85d-4b12-b7f3-548e1bd147a2 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=f610aae7-53b6-41d5-a4d0-dc6cea834431 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj8012288.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj8012288.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j8012288 -n arohcp-prow-dbcs-j8012288 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8012288/providers/Microsoft.Network/dnszones/j8012288.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j8012288-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j8012288-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n  -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n  --set serviceAccountName=clusters-service \\\\\\n  --set environment=arohcpprow \\\\\\n  --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n  --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n  --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n  --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n  --set tlsCertificatesEnabled=true \\\\\\n  --set denyAssignments=disabled \\\\\\n  --set tenantId=${TENANT_ID} \\\\\\n  --set region=westus3 \\\\\\n  --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n  --set replicas=3 \\\\\\n  --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n  --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n  --set imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set global.imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n  --set batchProcessesDryRun=true \\\\\\n  --set batchProcesses=\\\"\\\" \\\\\\n  --set fpaCertName=firstPartyCert2 \\\\\\n  --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n  --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n  --set shard.consumerName=\\\"hcp-underlay-j8012288-mgmt-1\\\" \\\\\\n  --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n  --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultMiClientId=\\\"542ca1fd-842a-439e-8779-f6f37f986389\\\" \\\\\\n  --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n  --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n  --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j8012288-mgmt-1'))\\\") \\\\\\n  --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8012288-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j8012288-mgmt-1 \\\\\\n  --set databaseHost=${DB_HOST} \\\\\\n  --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n  --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n  --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n  --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n  --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n  --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n  --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n  --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n  --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n  --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n  --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n  --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n  --set tracing.address=http://ingest.observability:4318 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n  --set provisionShardClusterLimit=100 \\\\\\n  --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n  --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n  --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n  --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n  --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n  --set azureOperatorsMI.roleSetName=dev \\\\\\n  --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036143094902034433","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4584,"pr_state":"closed","pr_sha":"4999eef2bb460e178d184cded32a17edd3e2ffea","final_merged_sha":"4999eef2bb460e178d184cded32a17edd3e2ffea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-23T18:05:35Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f31ac15f881c4d8f75c5f1fdc9eff07581a4ce2581cdea28eb0987a0d273d54d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036143094902034433","occurred_at":"2026-03-23T18:05:35Z","failed_at":"provision","signature_id":"5058d0fd74c81872666f058ed001e141a7fb54cc9c8248d41819c9fdda7930a5","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RouteMonitorOperator/management/deploy","test_suite":"step graph","failure_text":"time=2026-03-23T18:50:04.100Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-23T18:50:04.664Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy values=\"map[imageDigestBlackbox:sha256:b04a9fef4fa086a02fc7fcd8dcdbc4b7b35cc30cdee860fdc6a19dd8b208d63e imageDigestManager:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageDigestOperator:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageRegistryBlackbox:arohcpsvcdev.azurecr.io imageRegistryManager:arohcpsvcdev.azurecr.io imageRegistryOperator:arohcpsvcdev.azurecr.io imageRepositoryBlackbox:prometheus/blackbox-exporter imageRepositoryManager:app-sre/route-monitor-operator imageRepositoryOperator:app-sre/route-monitor-operator]\"\ntime=2026-03-23T18:50:04.664Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-23T18:50:05.147Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace openshift-route-monitor-operator: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a36b175ede41d233d6b695b53eeb0202c33fa8abba1121a12200be20e321df3a","failure_pattern":"failed to apply namespace openshift-route-monitor-operator: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace openshift-route-monitor-operator: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a36b175ede41d233d6b695b53eeb0202c33fa8abba1121a12200be20e321df3a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036141935227310080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4482,"pr_state":"closed","pr_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","final_merged_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-23T18:03:31Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"93327595d987bd6021e964e26c12449489991cd2919c588aa40c80c60337eb4e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036141935227310080","occurred_at":"2026-03-23T18:03:31Z","failed_at":"e2e","signature_id":"a3efdf650cd349146cddd9947be72415bd66f0fd3942f55bce1b5f096cc7632d","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:125]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0000dcb00\u003e: \n    failed to start credential request: POST https://rp.j7310080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-v25j69/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://rp.j7310080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-v25j69/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000912ff0\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"b163422a-6a86-40c3-af8a-af46b3a6eb36\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 19:13:37 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"64\"],\n                    \"Server\": [\"istio-envoy\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0000dc580\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j7310080.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-v25j69/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"Authorization\": [\"redacted\"],\n                        \"Accept\": [\"application/json\"],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n                        ],\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"c...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:125]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0000dcb00\u003e: \n    failed to start credential request: POST https://rp.j7310080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-v25j69/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://rp.j7310080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-v25j69/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000912ff0\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"b163422a-6a86-40c3-af8a-af46b3a6eb36\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 19:13:37 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"64\"],\n                    \"Server\": [\"istio-envoy\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0000dc580\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j7310080.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-v25j69/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"Authorization\": [\"redacted\"],\n                        \"Accept\": [\"application/json\"],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n                        ],\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"c...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036138003327356928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4585,"pr_state":"closed","pr_sha":"5ed7b324786bb326a526a06fec3e2fdc2ceec01f","final_merged_sha":"5ed7b324786bb326a526a06fec3e2fdc2ceec01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-23T17:40:35Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2dee1d860b9f3e55ba0b95b9ef0b3a7e51c68a7f115a147c0d43a75ca25d7467","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036138003327356928","occurred_at":"2026-03-23T17:40:35Z","failed_at":"e2e","signature_id":"abbe14d66a0b21865cf03a22e4a63963a11b2e33741e0462ae3bb49d8d26f6b6","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:167]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e04780\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-rl8trq\" to finish creating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-rl8trq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool np-1: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-rl8trq\\\" to finish creating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-rl8trq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000e04760\u003e{\n            msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-rl8trq\\\" to finish creating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-rl8trq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00155d230\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 500,\n                RawResponse: {\n                    Status: \"500 Internal Server Error\",\n                    StatusCode: 500,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Error-Code\": [\n                            \"InternalServerError\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"df7e757d-5e64-40a7-ade1-a33a21eb48eb\",\n                        ],\n                        \"Date\": [\n                            \"Mon, 23 Mar 2026 18:53:15 GMT\",\n                        ],\n                        \"Content-Length\": [\"107\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"2823\"],\n                        \"Server\": [\"istio-envoy\"],\n                        \"Content-Type\": [\"application/json\"],\n                    },\n                    Body: \u003c*exported.nopClosingBytesReader | 0xc000e04600\u003e{\n                        s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                        i: 0,\n                    },\n                    ContentLength: 107,\n                    TransferEncoding: nil,\n                    Close: false,\n                    Uncompressed: false,\n                    Trailer: nil,\n                    Request: {\n                        Method: \"GET\",\n                        URL: {\n                            Scheme: \"https\",\n                            Opaque: \"\",\n                            User: nil,\n                            Host: \"rp.j7356928.hcpsvc.osadev.cloud\",\n                            Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:167]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e04780\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-rl8trq\" to finish creating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-rl8trq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool np-1: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-rl8trq\\\" to finish creating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-rl8trq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000e04760\u003e{\n            msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-rl8trq\\\" to finish creating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-rl8trq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00155d230\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 500,\n                RawResponse: {\n                    Status: \"500 Internal Server Error\",\n                    StatusCode: 500,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Error-Code\": [\n                            \"InternalServerError\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"df7e757d-5e64-40a7-ade1-a33a21eb48eb\",\n                        ],\n                        \"Date\": [\n                            \"Mon, 23 Mar 2026 18:53:15 GMT\",\n                        ],\n                        \"Content-Length\": [\"107\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"2823\"],\n                        \"Server\": [\"istio-envoy\"],\n                        \"Content-Type\": [\"application/json\"],\n                    },\n                    Body: \u003c*exported.nopClosingBytesReader | 0xc000e04600\u003e{\n                        s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                        i: 0,\n                    },\n                    ContentLength: 107,\n                    TransferEncoding: nil,\n                    Close: false,\n                    Uncompressed: false,\n                    Trailer: nil,\n                    Request: {\n                        Method: \"GET\",\n                        URL: {\n                            Scheme: \"https\",\n                            Opaque: \"\",\n                            User: nil,\n                            Host: \"rp.j7356928.hcpsvc.osadev.cloud\",\n                            Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"532f376e26357b5a3b5e42adf01aa2aa904ebf16eb74bc2e1b9f8d62b0e247d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036138003327356928","occurred_at":"2026-03-23T17:40:35Z","failed_at":"e2e","signature_id":"b733d614711e9d558aedde83597b9305609aef4d1e62e7e0cccb2bd0a8f5da35","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009d0dc0\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-pv6mwr\" to finish updating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-pv6mwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/nodePools/np-one-node\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-pv6mwr\\\" to finish updating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-pv6mwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/nodePools/np-one-node\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00095d320\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"2860\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"cf637623-342c-4b03-8376-ec174e18500b\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 18:53:15 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0009d0d00\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"GET\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j7356928.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-pv6mwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/nodePools/np-one-node\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T18:52:47Z\\\"}\",\n                        ],\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (g...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009d0dc0\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-pv6mwr\" to finish updating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-pv6mwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/nodePools/np-one-node\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-pv6mwr\\\" to finish updating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-pv6mwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/nodePools/np-one-node\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00095d320\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"2860\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"cf637623-342c-4b03-8376-ec174e18500b\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 18:53:15 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0009d0d00\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"GET\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j7356928.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-pv6mwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/nodePools/np-one-node\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T18:52:47Z\\\"}\",\n                        ],\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (g...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5d025250117c5032b5855e1c186d5f4451cf7616a1471dc48240ae4c946ebc31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036138003327356928","occurred_at":"2026-03-23T17:40:35Z","failed_at":"e2e","signature_id":"090aecca14d0110f92ee92fdbd4b2d4ca8df58147fdddfac70fddc1ce13749e0","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:152]: Unexpected error:\n    \u003c*azidentity.AuthenticationFailedError | 0xc000995b90\u003e: \n    AzureCLICredential: signal: killed\n    {\n        RawResponse: nil,\n        credType: \"AzureCLICredential\",\n        message: \"signal: killed\",\n        omitResponse: false,\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:152]: Unexpected error:\n    \u003c*azidentity.AuthenticationFailedError | 0xc000995b90\u003e: \n    AzureCLICredential: signal: killed\n    {\n        RawResponse: nil,\n        credType: \"AzureCLICredential\",\n        message: \"signal: killed\",\n        omitResponse: false,\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"289a1415e3b418fc2753ab77620c5ae2ddc98c379e43280b17384eb6467d4ec2","failure_pattern":"AzureCLICredential: signal: killed","search_query":"occurred"}},{"row_id":"f892da1c2b2bb1986c4c14ced86c0795b1728f3d1d7a4402c05bf5a94aa01c51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036138003327356928","occurred_at":"2026-03-23T17:40:35Z","failed_at":"e2e","signature_id":"19d68b4fc73cb58b3a6d6329d1b4ddcd055320ccf0bb9733cf110a2c13b04b9a","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:139]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008c0c00\u003e: \n    failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-rhqc66\" to finish creating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-rhqc66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/nodePools/ephemeral-np\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed waiting for nodepool=\\\"ephemeral-np\\\" for cluster \\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-rhqc66\\\" to finish creating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-rhqc66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/nodePools/ephemeral-np\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000937440\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"e2e35a32-25dd-4716-bde4-6f5b2d4421c3\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 18:53:09 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"3006\"],\n                    \"Server\": [\"istio-envoy\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0008c0b40\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"GET\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j7356928.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-rhqc66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/nodePools/ephemeral-np\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2025-12-23-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T18:52:45Z\\\"}\",\n                        ],\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"Authorization\": [\"redacted\"],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:139]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008c0c00\u003e: \n    failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-rhqc66\" to finish creating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-rhqc66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/nodePools/ephemeral-np\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed waiting for nodepool=\\\"ephemeral-np\\\" for cluster \\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-rhqc66\\\" to finish creating: GET https://rp.j7356928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-rhqc66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/nodePools/ephemeral-np\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000937440\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"e2e35a32-25dd-4716-bde4-6f5b2d4421c3\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 18:53:09 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"3006\"],\n                    \"Server\": [\"istio-envoy\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0008c0b40\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"GET\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j7356928.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-rhqc66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/nodePools/ephemeral-np\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2025-12-23-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T18:52:45Z\\\"}\",\n                        ],\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"Authorization\": [\"redacted\"],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","289a1415e3b418fc2753ab77620c5ae2ddc98c379e43280b17384eb6467d4ec2"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036141622709719040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4586,"pr_state":"closed","pr_sha":"0d693572f01e0ca9ac068302880fa6634c11b39b","final_merged_sha":"89f77ed8135cc386d33aa903c6bdcc1e930b7268","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T17:39:41Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"bfd67263e145ab212c673d39e01c052159d933e3a1e4c69ac841bb8c44b18a2a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036141622709719040","occurred_at":"2026-03-23T17:39:41Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036136990230974466","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"bcf40750687ac69960989c50c78db02c28365d74","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T17:37:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6f18212cdc3e973d5d9ca74c4f43b0e13ac24c6f6d71443f3b2c2acd723e32f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036136990230974466","occurred_at":"2026-03-23T17:37:56Z","failed_at":"provision","signature_id":"a01c1d6a1a6d066cee6dc5d8d7835a6d9f7b1bf4b13cf546bd5aa81c6e406057","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-23T18:04:49.476Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n  Kind: Shell\\n  Command: make deploy\\n\"\ntime=2026-03-23T18:05:15.919Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy step=deploy error=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=32fcf37b-af9e-40c7-9ca0-7b55a80a7367 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=833a9afd-6f6e-43e3-8229-44e0623dfc7f \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj0974466.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj0974466.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j0974466 -n arohcp-prow-dbcs-j0974466 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0974466/providers/Microsoft.Network/dnszones/j0974466.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j0974466-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j0974466-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n  -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n  --set serviceAccountName=clusters-service \\\\\\n  --set environment=arohcpprow \\\\\\n  --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n  --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n  --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n  --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n  --set tlsCertificatesEnabled=true \\\\\\n  --set denyAssignments=disabled \\\\\\n  --set tenantId=${TENANT_ID} \\\\\\n  --set region=westus3 \\\\\\n  --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n  --set replicas=3 \\\\\\n  --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n  --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n  --set imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set global.imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n  --set batchProcessesDryRun=true \\\\\\n  --set batchProcesses=\\\"\\\" \\\\\\n  --set fpaCertName=firstPartyCert2 \\\\\\n  --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n  --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n  --set shard.consumerName=\\\"hcp-underlay-j0974466-mgmt-1\\\" \\\\\\n  --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n  --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultMiClientId=\\\"afa01b7c-2266-4069-b0ee-3f2e2e7482d0\\\" \\\\\\n  --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n  --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n  --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j0974466-mgmt-1'))\\\") \\\\\\n  --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0974466-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j0974466-mgmt-1 \\\\\\n  --set databaseHost=${DB_HOST} \\\\\\n  --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n  --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n  --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n  --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n  --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n  --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n  --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n  --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n  --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n  --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n  --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n  --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n  --set tracing.address=http://ingest.observability:4318 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n  --set provisionShardClusterLimit=100 \\\\\\n  --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n  --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n  --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n  --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n  --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n  --set azureOperatorsMI.roleSetName=dev \\\\\\n  --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: 1 error occurred:\\n\\t* Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.40:15017, code 500: 500 Internal Server Error\\n\\n\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"\ntime=2026-03-23T18:06:15.920Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n  Kind: Shell\\n  Command: make deploy\\n\"\ntime=2026-03-23T18:06:30.844Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=32fcf37b-af9e-40c7-9ca0-7b55a80a7367 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=833a9afd-6f6e-43e3-8229-44e0623dfc7f \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj0974466.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj0974466.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j0974466 -n arohcp-prow-dbcs-j0974466 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0974466/providers/Microsoft.Network/dnszones/j0974466.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j0974466-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j0974466-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n  -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n  --set serviceAccountName=clusters-service \\\\\\n  --set environment=arohcpprow \\\\\\n  --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n  --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n  --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n  --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n  --set tlsCertificatesEnabled=true \\\\\\n  --set denyAssignments=disabled \\\\\\n  --set tenantId=${TENANT_ID} \\\\\\n  --set region=westus3 \\\\\\n  --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n  --set replicas=3 \\\\\\n  --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n  --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n  --set imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set global.imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n  --set batchProcessesDryRun=true \\\\\\n  --set batchProcesses=\\\"\\\" \\\\\\n  --set fpaCertName=firstPartyCert2 \\\\\\n  --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n  --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n  --set shard.consumerName=\\\"hcp-underlay-j0974466-mgmt-1\\\" \\\\\\n  --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n  --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultMiClientId=\\\"afa01b7c-2266-4069-b0ee-3f2e2e7482d0\\\" \\\\\\n  --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n  --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n  --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j0974466-mgmt-1'))\\\") \\\\\\n  --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0974466-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j0974466-mgmt-1 \\\\\\n  --set databaseHost=${DB_HOST} \\\\\\n  --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n  --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n  --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n  --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n  --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n  --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n  --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n  --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n  --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n  --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n  --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n  --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n  --set tracing.address=http://ingest.observability:4318 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n  --set provisionShardClusterLimit=100 \\\\\\n  --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n  --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n  --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n  --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n  --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n  --set azureOperatorsMI.roleSetName=dev \\\\\\n  --set deployment.zoneCount=4\\nnamespace/clusters-service unchanged\\nnamespace/clusters-service not labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nError: UPGRADE FAILED: could not get information about the resource: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036140144372748288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"c86c5ea8623e834f6577d715795f8cf706e04c3f","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T17:22:12Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"20f5a4ca0849598e7858fb3d26b216ffa48eef57c93fe4defb13a97bb6b04adf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036140144372748288","occurred_at":"2026-03-23T17:22:12Z","failed_at":"provision","signature_id":"b9737aa3b12421b89fa07bdc4b32a28e7bab512bc99985bf2c5ebd0526881fbf","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-crds","test_suite":"step graph","failure_text":"time=2026-03-23T18:21:44.825Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds description=\"Step deploy-mce-crds\\n Kind: Helm\\n\"\ntime=2026-03-23T18:21:45.294Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds values=map[]\ntime=2026-03-23T18:21:45.294Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-23T18:21:45.837Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds namespace=multicluster-engine\ntime=2026-03-23T18:21:46.324Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds dryRun=false\ntime=2026-03-23T18:21:46.324Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-23T18:21:46.884Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds numVersions=0\ntime=2026-03-23T18:21:47.081Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds err=\"\\\"mce-crds\\\" has no deployed releases\"\ntime=2026-03-23T18:21:47.081Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-23T18:21:47.081Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-23T18:21:47.212Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds err=\"failed to get status for release mce-crds: release: not found\"\ntime=2026-03-23T18:21:47.212Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-23T18:21:47.212Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds err=\"error running Helm release deployment Step, failed to deploy helm release: \\\"mce-crds\\\" has no deployed releases\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dcc014bcb1baf89b15ca4b0a32b52a329377b68f8bdd88c2038f799238cab5fc","failure_pattern":"\"mce-crds\" has no deployed releases","search_query":"error running Helm release deployment Step, failed to deploy helm release: \\\"mce-crds\\\" has no deployed releases"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dcc014bcb1baf89b15ca4b0a32b52a329377b68f8bdd88c2038f799238cab5fc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036125539793637376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4585,"pr_state":"closed","pr_sha":"5ed7b324786bb326a526a06fec3e2fdc2ceec01f","final_merged_sha":"5ed7b324786bb326a526a06fec3e2fdc2ceec01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-23T16:58:52Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2c591fa323c6f5ffeb647f4a516af54bd4ade0f7e214e5d529bb3c0d75e6e56c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036125539793637376","occurred_at":"2026-03-23T16:58:52Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-23T17:20:09.109Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n  Kind: Shell\\n  Command: make deploy\\n\"\ntime=2026-03-23T17:20:27.463Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=0f8a3c87-daee-4534-8adb-978d10f3fbc3 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=a9d7c021-5881-4d2e-9250-9944da142893 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj3637376.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj3637376.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j3637376 -n arohcp-prow-dbcs-j3637376 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3637376/providers/Microsoft.Network/dnszones/j3637376.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j3637376-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j3637376-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n  -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n  --set serviceAccountName=clusters-service \\\\\\n  --set environment=arohcpprow \\\\\\n  --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n  --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n  --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n  --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n  --set tlsCertificatesEnabled=true \\\\\\n  --set denyAssignments=disabled \\\\\\n  --set tenantId=${TENANT_ID} \\\\\\n  --set region=westus3 \\\\\\n  --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n  --set replicas=3 \\\\\\n  --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n  --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n  --set imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set global.imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n  --set batchProcessesDryRun=true \\\\\\n  --set batchProcesses=\\\"\\\" \\\\\\n  --set fpaCertName=firstPartyCert2 \\\\\\n  --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n  --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n  --set shard.consumerName=\\\"hcp-underlay-j3637376-mgmt-1\\\" \\\\\\n  --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n  --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultMiClientId=\\\"25a5d469-6151-4fcb-b6ab-3254f7c5f6f8\\\" \\\\\\n  --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n  --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n  --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j3637376-mgmt-1'))\\\") \\\\\\n  --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3637376-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j3637376-mgmt-1 \\\\\\n  --set databaseHost=${DB_HOST} \\\\\\n  --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n  --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n  --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n  --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n  --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n  --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n  --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n  --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n  --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n  --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n  --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n  --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n  --set tracing.address=http://ingest.observability:4318 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n  --set provisionShardClusterLimit=100 \\\\\\n  --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n  --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n  --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n  --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n  --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n  --set azureOperatorsMI.roleSetName=dev \\\\\\n  --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: could not get server version from Kubernetes: the server has asked for the client to provide credentials\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036123432554336256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4584,"pr_state":"closed","pr_sha":"4999eef2bb460e178d184cded32a17edd3e2ffea","final_merged_sha":"4999eef2bb460e178d184cded32a17edd3e2ffea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-23T16:50:30Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"12850f1acb9188b00e0921d8a6bdb8ddab35a57f01f578d32a00d96e1001fc68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036123432554336256","occurred_at":"2026-03-23T16:50:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4518/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036121566634315776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4518,"pr_state":"closed","pr_sha":"b6a19c9b7bc7779f36da9354dff2ffd3825c6c80","final_merged_sha":"c68c3104b7e5c84dff8c2653a50758b036821419","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T16:43:05Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"1627626381d407274c5d83e2562d50985d9dbec23e0b14eb7b57a6037a5a19e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4518/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036121566634315776","occurred_at":"2026-03-23T16:43:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036115698672996352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4556,"pr_state":"open","pr_sha":"047f4394fbfb3ee76f54b58c25ba5019edf9b456","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T16:19:26Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"49cf11cf6fee4b9dd15b3959407d68f1f80ec77f1d6660df4e69aeaf3a13ac40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036115698672996352","occurred_at":"2026-03-23T16:19:26Z","failed_at":"e2e","signature_id":"2e5aab3b9af8a8472e79dfc349c0e73808e80353e848acfe0cca69e6aa0f29d7","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:105]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00011ef20\u003e: \n    failed to start credential request: POST https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rsp6lc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-g75n2h/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rsp6lc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-g75n2h/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001440510\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"fad802c6-cf1f-464a-8144-b9fa410d06da\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 17:14:23 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"76\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00011eca0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j2996352.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rsp6lc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-g75n2h/requestAdminCredential\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"Accept\": [\"application/json\"],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n                        ],\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T17:14:13...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:105]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00011ef20\u003e: \n    failed to start credential request: POST https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rsp6lc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-g75n2h/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rsp6lc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-g75n2h/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001440510\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"fad802c6-cf1f-464a-8144-b9fa410d06da\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 17:14:23 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"76\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00011eca0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j2996352.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rsp6lc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-g75n2h/requestAdminCredential\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"Accept\": [\"application/json\"],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n                        ],\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T17:14:13...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5b155761f53ef13e932598af6b70fb49f7300258528f080ea29925a28ce9440f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036115698672996352","occurred_at":"2026-03-23T16:19:26Z","failed_at":"e2e","signature_id":"c227a5371665d13bf387626ee852b2475c72e1a43a126ee7d148e11dcaab5d51","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:115]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0006be500\u003e: \n    failed to start credential request: POST https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-8ftcvf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-8ftcvf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001514810\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"107\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"6b966dbe-edf7-4bda-873e-881fcfddda7a\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 17:14:18 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0006be440\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j2996352.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-8ftcvf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster/requestAdminCredential\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"Authorization\": [\"redacted\"],\n                        \"Accept\": [\"application/json\"],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n                        ],\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T17:14:09Z\\\"}\",\n                        ],\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:115]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0006be500\u003e: \n    failed to start credential request: POST https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-8ftcvf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-8ftcvf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001514810\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"107\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"6b966dbe-edf7-4bda-873e-881fcfddda7a\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 17:14:18 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0006be440\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j2996352.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-8ftcvf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster/requestAdminCredential\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"Authorization\": [\"redacted\"],\n                        \"Accept\": [\"application/json\"],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n                        ],\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T17:14:09Z\\\"}\",\n                        ],\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d18109ca0c69e9478939a9359886249543eabed0f277368faab92b59f2e77ca9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036115698672996352","occurred_at":"2026-03-23T16:19:26Z","failed_at":"e2e","signature_id":"e59ec1b5e5c44a16b18bf13979b25f609ba43681355d023123306a4493f25c89","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000984360\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-9pnhv4: failed waiting for cluster=\"cluster-zstream-4-21-9pnhv4\" in resourcegroup=\"rg-zstream-upgrade-4-21-7nx4d8\" to finish creating: GET https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-7nx4d8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9pnhv4\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-21-9pnhv4: failed waiting for cluster=\\\"cluster-zstream-4-21-9pnhv4\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-7nx4d8\\\" to finish creating: GET https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-7nx4d8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9pnhv4\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000984300\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-9pnhv4\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-7nx4d8\\\" to finish creating: GET https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-7nx4d8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9pnhv4\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0013002d0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 500,\n                RawResponse: {\n                    Status: \"500 Internal Server Error\",\n                    StatusCode: 500,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"107\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"3511\"],\n                        \"Server\": [\"istio-envoy\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InternalServerError\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"5ffba58e-5961-48d0-9dab-a5a07d23df06\",\n                        ],\n                        \"Date\": [\n                            \"Mon, 23 Mar 2026 17:23:13 GMT\",\n                        ],\n                    },\n                    Body: \u003c*exported.nopClosingBytesReader | 0xc000984020\u003e{\n                        s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                        i: 0,\n                    },\n                    ContentLength: 107,\n                    TransferEncoding: nil,\n                    Close: false,\n                    Uncompressed: false,\n                    Trailer: nil,\n                    Request: {\n                        Method: \"GET\",\n                        URL: {\n                            Scheme: \"https\",\n                            Opaque: \"\",\n                            User: nil,\n                            Host: \"rp.j2996352.hcpsvc.osadev.cloud\",\n                            Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstr...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000984360\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-9pnhv4: failed waiting for cluster=\"cluster-zstream-4-21-9pnhv4\" in resourcegroup=\"rg-zstream-upgrade-4-21-7nx4d8\" to finish creating: GET https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-7nx4d8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9pnhv4\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-21-9pnhv4: failed waiting for cluster=\\\"cluster-zstream-4-21-9pnhv4\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-7nx4d8\\\" to finish creating: GET https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-7nx4d8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9pnhv4\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000984300\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-9pnhv4\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-7nx4d8\\\" to finish creating: GET https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-7nx4d8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9pnhv4\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0013002d0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 500,\n                RawResponse: {\n                    Status: \"500 Internal Server Error\",\n                    StatusCode: 500,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"107\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"3511\"],\n                        \"Server\": [\"istio-envoy\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InternalServerError\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"5ffba58e-5961-48d0-9dab-a5a07d23df06\",\n                        ],\n                        \"Date\": [\n                            \"Mon, 23 Mar 2026 17:23:13 GMT\",\n                        ],\n                    },\n                    Body: \u003c*exported.nopClosingBytesReader | 0xc000984020\u003e{\n                        s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                        i: 0,\n                    },\n                    ContentLength: 107,\n                    TransferEncoding: nil,\n                    Close: false,\n                    Uncompressed: false,\n                    Trailer: nil,\n                    Request: {\n                        Method: \"GET\",\n                        URL: {\n                            Scheme: \"https\",\n                            Opaque: \"\",\n                            User: nil,\n                            Host: \"rp.j2996352.hcpsvc.osadev.cloud\",\n                            Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstr...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ee1174b622ef7f752ddd57a35486608327bc8a07da52ce959127e3ee8d538aa5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036115698672996352","occurred_at":"2026-03-23T16:19:26Z","failed_at":"e2e","signature_id":"c8c2e8a71f20b531b0b0f2c962608b890085cc6617bdb7345eb5ffcea229bf45","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:418]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc0010b8000\u003e: \n    GET https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-snv4ht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"InternalServerError\",\n        StatusCode: 500,\n        RawResponse: {\n            Status: \"500 Internal Server Error\",\n            StatusCode: 500,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"X-Envoy-Upstream-Service-Time\": [\"2610\"],\n                \"Server\": [\"istio-envoy\"],\n                \"Content-Type\": [\"application/json\"],\n                \"X-Ms-Error-Code\": [\n                    \"InternalServerError\",\n                ],\n                \"X-Ms-Request-Id\": [\n                    \"b7d20e26-db8e-4737-b04f-c04d91aaed58\",\n                ],\n                \"Date\": [\n                    \"Mon, 23 Mar 2026 17:09:57 GMT\",\n                ],\n                \"Content-Length\": [\"107\"],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc0009c4f80\u003e{\n                s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                i: 0,\n            },\n            ContentLength: 107,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"GET\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"rp.j2996352.hcpsvc.osadev.cloud\",\n                    Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-snv4ht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\",\n                    RawPath: \"\",\n                    OmitHost: false,\n                    ForceQuery: false,\n                    RawQuery: \"api-version=2024-06-10-preview\",\n                    Fragment: \"\",\n                    RawFragment: \"\",\n                },\n                Proto: \"HTTP/1.1\",\n                ProtoMajor: 1,\n                ProtoMinor: 1,\n                Header: {\n                    \"User-Agent\": [\n                        \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n                    ],\n                    \"X-Ms-Arm-Resource-System-Data\": [\n                        \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T17:09:36Z\\\"}\",\n                    ],\n                    \"X-Ms-Identity-Url\": [\n                        \"https://dummyhost.identity.azure.net\",\n                    ],\n                    \"Authorization\": [\"redacted\"],\n                },\n                Body: nil,\n                GetBody: nil,\n                ContentLength: 0,\n                TransferEncoding: nil,\n                Close: false,\n                Host: \"rp.j2996352.hcpsvc.osadev.cloud\",\n                Form: nil,\n                PostForm: nil,\n                MultipartForm: nil,\n                Trailer: nil,\n                RemoteAddr: \"\",\n                RequestURI: \"\",\n                TLS: nil,\n                Cancel: nil,\n                Response: nil,\n                Pattern: \"\",\n                ctx: \u003csuppressed context\u003e,\n                pat: nil,\n                matches: nil,\n                otherValues: nil,\n            },\n            TLS: {\n                Version: 772,\n                HandshakeComplete: true,\n                DidResume: false,\n                CipherSuite: 4865,\n                CurveID: 29,\n                NegotiatedProtocol: \"h2\",\n                NegotiatedProtocolIsMutual: true,\n                ServerName: \"rp.j2996352.hcpsvc.osadev.cloud\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:418]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc0010b8000\u003e: \n    GET https://rp.j2996352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-snv4ht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"InternalServerError\",\n        StatusCode: 500,\n        RawResponse: {\n            Status: \"500 Internal Server Error\",\n            StatusCode: 500,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"X-Envoy-Upstream-Service-Time\": [\"2610\"],\n                \"Server\": [\"istio-envoy\"],\n                \"Content-Type\": [\"application/json\"],\n                \"X-Ms-Error-Code\": [\n                    \"InternalServerError\",\n                ],\n                \"X-Ms-Request-Id\": [\n                    \"b7d20e26-db8e-4737-b04f-c04d91aaed58\",\n                ],\n                \"Date\": [\n                    \"Mon, 23 Mar 2026 17:09:57 GMT\",\n                ],\n                \"Content-Length\": [\"107\"],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc0009c4f80\u003e{\n                s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                i: 0,\n            },\n            ContentLength: 107,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"GET\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"rp.j2996352.hcpsvc.osadev.cloud\",\n                    Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-snv4ht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\",\n                    RawPath: \"\",\n                    OmitHost: false,\n                    ForceQuery: false,\n                    RawQuery: \"api-version=2024-06-10-preview\",\n                    Fragment: \"\",\n                    RawFragment: \"\",\n                },\n                Proto: \"HTTP/1.1\",\n                ProtoMajor: 1,\n                ProtoMinor: 1,\n                Header: {\n                    \"User-Agent\": [\n                        \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n                    ],\n                    \"X-Ms-Arm-Resource-System-Data\": [\n                        \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T17:09:36Z\\\"}\",\n                    ],\n                    \"X-Ms-Identity-Url\": [\n                        \"https://dummyhost.identity.azure.net\",\n                    ],\n                    \"Authorization\": [\"redacted\"],\n                },\n                Body: nil,\n                GetBody: nil,\n                ContentLength: 0,\n                TransferEncoding: nil,\n                Close: false,\n                Host: \"rp.j2996352.hcpsvc.osadev.cloud\",\n                Form: nil,\n                PostForm: nil,\n                MultipartForm: nil,\n                Trailer: nil,\n                RemoteAddr: \"\",\n                RequestURI: \"\",\n                TLS: nil,\n                Cancel: nil,\n                Response: nil,\n                Pattern: \"\",\n                ctx: \u003csuppressed context\u003e,\n                pat: nil,\n                matches: nil,\n                otherValues: nil,\n            },\n            TLS: {\n                Version: 772,\n                HandshakeComplete: true,\n                DidResume: false,\n                CipherSuite: 4865,\n                CurveID: 29,\n                NegotiatedProtocol: \"h2\",\n                NegotiatedProtocolIsMutual: true,\n                ServerName: \"rp.j2996352.hcpsvc.osadev.cloud\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036113067649011712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4585,"pr_state":"closed","pr_sha":"5ed7b324786bb326a526a06fec3e2fdc2ceec01f","final_merged_sha":"5ed7b324786bb326a526a06fec3e2fdc2ceec01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-23T16:07:38Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"efbcad0bf34b434f41071b78e70af5a9627c9379e6be44df34f8d8661a271173","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036113067649011712","occurred_at":"2026-03-23T16:07:38Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-23T16:49:01.435Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n  Kind: Shell\\n  Command: make deploy\\n\"\ntime=2026-03-23T16:49:18.472Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=a1860c91-e1c4-4628-8c7a-e39af585e22f \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=68990b8b-945a-40a4-ad14-c0f8ee107135 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj9011712.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj9011712.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j9011712 -n arohcp-prow-dbcs-j9011712 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9011712/providers/Microsoft.Network/dnszones/j9011712.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j9011712-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j9011712-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n  -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n  --set serviceAccountName=clusters-service \\\\\\n  --set environment=arohcpprow \\\\\\n  --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n  --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n  --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n  --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n  --set tlsCertificatesEnabled=true \\\\\\n  --set denyAssignments=disabled \\\\\\n  --set tenantId=${TENANT_ID} \\\\\\n  --set region=westus3 \\\\\\n  --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n  --set replicas=3 \\\\\\n  --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n  --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n  --set imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set global.imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n  --set batchProcessesDryRun=true \\\\\\n  --set batchProcesses=\\\"\\\" \\\\\\n  --set fpaCertName=firstPartyCert2 \\\\\\n  --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n  --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n  --set shard.consumerName=\\\"hcp-underlay-j9011712-mgmt-1\\\" \\\\\\n  --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n  --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultMiClientId=\\\"40329ede-b468-40bf-8e42-41153ed76e00\\\" \\\\\\n  --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n  --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n  --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j9011712-mgmt-1'))\\\") \\\\\\n  --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9011712-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j9011712-mgmt-1 \\\\\\n  --set databaseHost=${DB_HOST} \\\\\\n  --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n  --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n  --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n  --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n  --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n  --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n  --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n  --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n  --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n  --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n  --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n  --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n  --set tracing.address=http://ingest.observability:4318 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n  --set provisionShardClusterLimit=100 \\\\\\n  --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n  --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n  --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n  --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n  --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n  --set azureOperatorsMI.roleSetName=dev \\\\\\n  --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unable to continue with install: could not get information about the resource Deployment \\\"postgres-breakglass\\\" in namespace \\\"clusters-service\\\": Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036113167754465280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4579,"pr_state":"closed","pr_sha":"35bd2ace7e36ec862a86d56aaf86cdfd7bb32fba","final_merged_sha":"35bd2ace7e36ec862a86d56aaf86cdfd7bb32fba","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T16:03:20Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036111240723435520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4585,"pr_state":"closed","pr_sha":"277ca79574bd380465b514b7e6a17c0394f7a260","final_merged_sha":"5ed7b324786bb326a526a06fec3e2fdc2ceec01f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T15:59:24Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"dfe871b7c6a36dfa7083efdfcc77ba05478d60bd3fc623a666cf6d63d2347bc9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036111240723435520","occurred_at":"2026-03-23T15:59:24Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036109685848805376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4457,"pr_state":"open","pr_sha":"ec22b07e4d4dc37eb74d17df48d598540c139a1d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T15:55:32Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"473f29d01d50d9efd2c38e7a7f2be51e1bc4f98a88e6172ae2aec7aebbe36e4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036109685848805376","occurred_at":"2026-03-23T15:55:32Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-23T16:33:04.940Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n  Kind: Shell\\n  Command: make deploy\\n\"\ntime=2026-03-23T16:34:25.561Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=8ae5b76d-6bd0-44ed-9ecc-d9f925927be3 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=8009bd65-fd90-4d10-9d07-aca4690540d4 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj8805376.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj8805376.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j8805376 -n arohcp-prow-dbcs-j8805376 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8805376/providers/Microsoft.Network/dnszones/j8805376.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j8805376-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j8805376-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n  -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n  --set serviceAccountName=clusters-service \\\\\\n  --set environment=arohcpprow \\\\\\n  --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n  --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n  --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n  --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n  --set tlsCertificatesEnabled=true \\\\\\n  --set denyAssignments=disabled \\\\\\n  --set tenantId=${TENANT_ID} \\\\\\n  --set region=westus3 \\\\\\n  --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n  --set replicas=3 \\\\\\n  --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n  --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n  --set imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set global.imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n  --set batchProcessesDryRun=true \\\\\\n  --set batchProcesses=\\\"\\\" \\\\\\n  --set fpaCertName=firstPartyCert2 \\\\\\n  --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n  --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n  --set shard.consumerName=\\\"hcp-underlay-j8805376-mgmt-1\\\" \\\\\\n  --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n  --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultMiClientId=\\\"794c7e96-3278-4db5-9dce-f6a403211e98\\\" \\\\\\n  --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n  --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n  --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j8805376-mgmt-1'))\\\") \\\\\\n  --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8805376-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j8805376-mgmt-1 \\\\\\n  --set databaseHost=${DB_HOST} \\\\\\n  --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n  --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n  --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n  --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n  --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n  --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n  --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n  --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n  --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n  --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n  --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n  --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n  --set tracing.address=http://ingest.observability:4318 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n  --set provisionShardClusterLimit=100 \\\\\\n  --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n  --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n  --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n  --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n  --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n  --set azureOperatorsMI.roleSetName=dev \\\\\\n  --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036114153483014144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"closed","pr_sha":"7cddbd3cbc43f559f3df12084ec577bed405c99f","final_merged_sha":"4fe7725f48a2f3f1c0ac843b4434f502807b38e5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T15:48:29Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ba380efb15874c84513207fdf0663d3952c17087627699008688f222ba9170dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036114153483014144","occurred_at":"2026-03-23T15:48:29Z","failed_at":"provision","signature_id":"186999f428b36f71426b58a1596fcd1d6391598e85a630dbb52ced5c67a51707","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-crds","test_suite":"step graph","failure_text":"time=2026-03-23T16:39:59.671Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds description=\"Step deploy-mce-crds\\n Kind: Helm\\n\"\ntime=2026-03-23T16:40:00.185Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds values=map[]\ntime=2026-03-23T16:40:00.185Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-23T16:40:00.822Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace multicluster-engine: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b95b9c9bf7b4253eeec2aaaf028bc5abbf4cac62468310084bd0fb345750a5b","failure_pattern":"failed to apply namespace multicluster-engine: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace multicluster-engine: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7b95b9c9bf7b4253eeec2aaaf028bc5abbf4cac62468310084bd0fb345750a5b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036107799015985152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4584,"pr_state":"closed","pr_sha":"4999eef2bb460e178d184cded32a17edd3e2ffea","final_merged_sha":"4999eef2bb460e178d184cded32a17edd3e2ffea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-23T15:48:02Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"eeab5c04fa2d6ebdce82cdce7fa2e2ed988576f4434c71155f288c7942efac33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036107799015985152","occurred_at":"2026-03-23T15:48:02Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4563/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036107279027146752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4563,"pr_state":"open","pr_sha":"8ef79687ddc576575f993148b616af294979425b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T15:46:18Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1869c6714dff5b126459639e759b16c5b7d61bd35b8de3eba05322c41f37562a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4563/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036107279027146752","occurred_at":"2026-03-23T15:46:18Z","failed_at":"e2e","signature_id":"7451f79443cfc16f1475b85ca49bd7e239851255c66233d7f96bedeff5196af3","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00099eae0\u003e: \n    failed to start credential request: POST https://rp.j7146752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-chpshm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://rp.j7146752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-chpshm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00092b800\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"244bd667-fd47-429e-a668-8676c9666731\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 17:05:44 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"51\"],\n                    \"Server\": [\"istio-envoy\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00099ea20\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j7146752.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-chpshm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/requestAdminCredential\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"Accept\": [\"application/json\"],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n                        ],\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T17:05:30Z\\\"}\",\n       ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00099eae0\u003e: \n    failed to start credential request: POST https://rp.j7146752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-chpshm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://rp.j7146752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-chpshm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00092b800\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"244bd667-fd47-429e-a668-8676c9666731\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 17:05:44 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"51\"],\n                    \"Server\": [\"istio-envoy\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00099ea20\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j7146752.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-chpshm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/requestAdminCredential\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"Accept\": [\"application/json\"],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n                        ],\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T17:05:30Z\\\"}\",\n       ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"89442453eec625db61eac035d5fc64ad66430d137409132718a66aa2fa11d703","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4563/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036107279027146752","occurred_at":"2026-03-23T15:46:18Z","failed_at":"e2e","signature_id":"336be3a21bb10edac8f2b6e4e4f8b62930eb9ccdbdf9a1e5a56a2483a5e6c1e8","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008e8c00\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-j84dxz\" to finish updating: GET https://rp.j7146752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-j84dxz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/nodePools/np-one-node\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-j84dxz\\\" to finish updating: GET https://rp.j7146752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-j84dxz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/nodePools/np-one-node\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0008acf90\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"d119aea6-57cb-4bf2-9e82-0f2727d1ef38\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 17:11:05 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"2834\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0008e8b00\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"GET\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j7146752.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-j84dxz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/nodePools/np-one-node\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T17:10:44Z\\\"}\",\n                        ],\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (g...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008e8c00\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-j84dxz\" to finish updating: GET https://rp.j7146752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-j84dxz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/nodePools/np-one-node\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-j84dxz\\\" to finish updating: GET https://rp.j7146752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-j84dxz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/nodePools/np-one-node\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0008acf90\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"d119aea6-57cb-4bf2-9e82-0f2727d1ef38\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 17:11:05 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"2834\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0008e8b00\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"GET\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j7146752.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-j84dxz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/nodePools/np-one-node\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T17:10:44Z\\\"}\",\n                        ],\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (g...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036104920268017664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4583,"pr_state":"closed","pr_sha":"3128dcccf615b5f87038c48a9b25c6714da317b6","final_merged_sha":"39d00655b0878452b6e129a6301b0f285e262876","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T15:36:16Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ea8bc4024dbd7de78750836d7335fa6c1c460077d5c9c1698baf29a4265d5bdb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036104920268017664","occurred_at":"2026-03-23T15:36:16Z","failed_at":"provision","signature_id":"a6764bd9c591fad6425b0179e367b3053bf3f23f1c36edb2c5c1a6b78faeeaf4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/arobit","test_suite":"step graph","failure_text":"time=2026-03-23T16:16:36.862Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-03-23T16:16:37.551Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:prow-j8017664-mgmt-1 clusterType:mgmt dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:33bed43b09e3d2384c7661c74f6ed79ca53b0c25a3e28d77f139a7b9d50fbf1a pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit]] geoShortId:us kusto:map[enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net serviceLogsDatabase:ServiceLogs] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan: clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-prow-j8017664-mgmt-1 rpAccountName:placeholder rpNamespace: rpSan: rpSecretName:rplogs] image:map[digest:sha256:18d5476700adcd40e9ee2bc1cb62a1150ac1b5e696d7bad6adeea4da88c615db pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:27a07e3a-f481-439c-8e7a-e2a4ed160dda region:westus3 secretProvider:map[keyVault:ah-prow-mg-j8017664-1] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-03-23T16:16:37.551Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit\ntime=2026-03-23T16:16:37.865Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit namespace=arobit\ntime=2026-03-23T16:16:38.161Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit dryRun=false\ntime=2026-03-23T16:16:38.161Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit\ntime=2026-03-23T16:16:38.293Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit\ntime=2026-03-23T16:16:43.187Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit err=\"failed to create typed patch object (arobit/arobit-forwarder; apps/v1, Kind=DaemonSet): .spec.template.spec.containers[name=\\\"fluentbit\\\"].limits: field not declared in schema\"\ntime=2026-03-23T16:16:43.187Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit\ntime=2026-03-23T16:16:43.187Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit\ntime=2026-03-23T16:16:43.792Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit release=arobit namespace=arobit status=failed description=\"Release \\\"arobit\\\" failed: failed to create typed patch object (arobit/arobit-forwarder; apps/v1, Kind=DaemonSet): .spec.template.spec.containers[name=\\\"fluentbit\\\"].limits: field not declared in schema\" values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:prow-j8017664-mgmt-1 clusterType:mgmt dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:33bed43b09e3d2384c7661c74f6ed79ca53b0c25a3e28d77f139a7b9d50fbf1a pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit]] geoShortId:us kusto:map[enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net serviceLogsDatabase:ServiceLogs] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan: clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-prow-j8017664-mgmt-1 rpAccountName:placeholder rpNamespace: rpSan: rpSecretName:rplogs] image:map[digest:sha256:18d5476700adcd40e9ee2bc1cb62a1150ac1b5e696d7bad6adeea4da88c615db pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:27a07e3a-f481-439c-8e7a-e2a4ed160dda region:westus3 secretProvider:map[keyVault:ah-prow-mg-j8017664-1] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-03-23T16:16:43.793Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5SRwU7CQBCG732KSU80KQSoIQbDQY0ngwf1hPGw3R3qQjvTzE4hJjy8qaCgEdHrt98/mZ0/KlFBMHAjFgNMwBk1avISO5wv0OqtJzcOKp6KFLbozlT4iXCF9E5CbewHTp4iAID4mmnui6mp4xRiI5x77c5Z1kYcyp7F6U4vm6Ao91ziscAP6pUn56k4lXhAWXl7dPDXTXbylMkry78yl9ZyQ3oiEz1fRMsmRyFUDDftEUO0gfULCoL6CoOaqoYcdY1I0HFGscWdeNgfjrr9rDvMHgej8WA0zs5ncQK9HvzinGWzOEmiDSzYEyw9uYknQjmongn2jR9W/b3jaAO1cPu6XzSFmt1W/tuQFEouUqgwBFO0I7k9EOSvB593GOxbAAAA//8h+DJdoAIAAA==\"\ntime=2026-03-23T16:16:43.793Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit\ntime=2026-03-23T16:16:43.794Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to create typed patch object (arobit/arobit-forwarder; apps/v1, Kind=DaemonSet): .spec.template.spec.containers[name=\\\"fluentbit\\\"].limits: field not declared in schema\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b3054742c32446c91766fddbabfa349e00ce0ae344f5050d657f5db6e29b1","failure_pattern":"failed to create typed patch object (arobit/arobit-forwarder; apps/v1, Kind=DaemonSet): .spec.template.spec.containers[name=\"fluentbit\"].limits: field not declared in schema","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to create typed patch object (arobit/arobit-forwarder; apps/v1, Kind=DaemonSet): .spec.template.spec.containers[name=\\\"fluentbit\\\"].limits: field not declared in schema"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1a6b3054742c32446c91766fddbabfa349e00ce0ae344f5050d657f5db6e29b1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036110300935098368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"8c5b51a711f275765aed4c83bd6679a7d7245115","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T15:33:48Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"09479e3b697bff841bcad98c99579e0f62a3131655b600549c3dfa86b4fc63c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036110300935098368","occurred_at":"2026-03-23T15:33:48Z","failed_at":"provision","signature_id":"14e1139051b92a0259280ab4bba3ccc679edd3541b36e2204a8d4ec84f155427","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"step graph","failure_text":"time=2026-03-23T16:20:04.257Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-23T16:20:05.290Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:01038e7de14b78d702d2849c3aad72fd25903c4765af63cf16aa3398f5d5f2dd]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b7c83e5834c5e97ef17af96ddbcba9ed7062371a1db1a46171ff7ae1ae137aec] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:b7bd3a6bf3160b82af89c345c58b1fd3a936801c8c65fef7ed2111dc71c766a3]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:40959f79-762b-4855-8ace-e32c9a85699e]] prometheusSpec:map[externalLabels:map[cluster:prow-j5098368-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:eb5dcb5c4f6e229d6dfe17b492291e06bea7ba19f080d48a7afc663e4420386c] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j5098368-svc-ucq1.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-d742c6875b444f0288b9c15c476b06c1/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-23T16:20:05.290Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-23T16:20:05.382Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace prometheus: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/prometheus?fieldManager=templatize\u0026amp;force=false\\\\\\\": Post \\\\\\\"https://guard.69c164e47a8970000186b27d.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\\"crypto/rsa: verification error\\\\\\\" while trying to verify candidate authority certificate \\\\\\\"ca\\\\\\\")\\\") has prevented the request from succeeding (patch namespaces prometheus)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"38239b79b5b799e6cbc8dad16b9ce3a3e5236d97a0d80408b6d601146c6bf4fa","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["38239b79b5b799e6cbc8dad16b9ce3a3e5236d97a0d80408b6d601146c6bf4fa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036106147282292736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4554,"pr_state":"closed","pr_sha":"1ba8557e7aa3030dfdb65d2dedde1be704afe79f","final_merged_sha":"cab9df44f4aaadfa6301e36bed1fb8403e84a43b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T15:30:38Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2237ec816586e71790dfcfb148ab12093c34596badb32e82e882c9b5452d01e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036106147282292736","occurred_at":"2026-03-23T15:30:38Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-23T16:18:27.340Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-23T16:18:28.018Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:01038e7de14b78d702d2849c3aad72fd25903c4765af63cf16aa3398f5d5f2dd]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b7c83e5834c5e97ef17af96ddbcba9ed7062371a1db1a46171ff7ae1ae137aec] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:b7bd3a6bf3160b82af89c345c58b1fd3a936801c8c65fef7ed2111dc71c766a3]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:eb6fef22-193d-40a1-b6b4-e1f8efbd4b8a]] prometheusSpec:map[externalLabels:map[cluster:prow-j2292736-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j2292736-mgmt-1-2jsb.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-c53a8f96f02444a8877e72857b477fc0/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:eb5dcb5c4f6e229d6dfe17b492291e06bea7ba19f080d48a7afc663e4420386c] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j2292736-mgmt-1-2jsb.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-8b244679ca374250b457b2ec0755c578/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-23T16:18:28.019Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T16:18:28.313Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus namespace=prometheus\ntime=2026-03-23T16:18:28.658Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus dryRun=false\ntime=2026-03-23T16:18:28.658Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T16:18:28.831Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T16:20:41.651Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed post-install: warning: Hook post-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml failed: Unauthorized\"\ntime=2026-03-23T16:20:41.651Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T16:20:41.651Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T16:20:42.204Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed to get status for release arohcp-monitor: kubernetes cluster unreachable: the server has asked for the client to provide credentials\"\ntime=2026-03-23T16:20:42.204Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-23T16:20:42.204Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: warning: Hook post-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml failed: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83ff7ce859263e1194d23139f2381000b028b81f4625af1d7820f46493fa248a","failure_pattern":"failed post-install: warning: Hook post-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml failed: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: warning: Hook post-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml failed: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["83ff7ce859263e1194d23139f2381000b028b81f4625af1d7820f46493fa248a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036103264247746560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4556,"pr_state":"open","pr_sha":"047f4394fbfb3ee76f54b58c25ba5019edf9b456","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T15:26:00Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"056b6e9227be71c355e6d446dc90ad7b65fb7f111f2b6cd8b406f32f35b338e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036103264247746560","occurred_at":"2026-03-23T15:26:00Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-23T16:02:48.155Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n  Kind: Shell\\n  Command: make deploy\\n\"\ntime=2026-03-23T16:03:26.993Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=84b71e41-ec7c-4545-89ac-49bea8759c70 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=23f0e43a-8fea-4e65-b661-1bc7c496f80e \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj7746560.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj7746560.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j7746560 -n arohcp-prow-dbcs-j7746560 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7746560/providers/Microsoft.Network/dnszones/j7746560.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j7746560-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j7746560-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n  -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n  --set serviceAccountName=clusters-service \\\\\\n  --set environment=arohcpprow \\\\\\n  --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n  --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n  --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n  --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n  --set tlsCertificatesEnabled=true \\\\\\n  --set denyAssignments=disabled \\\\\\n  --set tenantId=${TENANT_ID} \\\\\\n  --set region=westus3 \\\\\\n  --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n  --set replicas=3 \\\\\\n  --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n  --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n  --set imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set global.imageDigest=sha256:e30ee0c2851431cc17f44a109f987cb845e77369036c9a8074f17b66163341c8 \\\\\\n  --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n  --set batchProcessesDryRun=true \\\\\\n  --set batchProcesses=\\\"\\\" \\\\\\n  --set fpaCertName=firstPartyCert2 \\\\\\n  --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n  --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n  --set shard.consumerName=\\\"hcp-underlay-j7746560-mgmt-1\\\" \\\\\\n  --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n  --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n  --set shard.cxSecretsKeyVaultMiClientId=\\\"9b43dd6b-be77-41f2-83c5-72b7b97417ee\\\" \\\\\\n  --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n  --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n  --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j7746560-mgmt-1'))\\\") \\\\\\n  --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7746560-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j7746560-mgmt-1 \\\\\\n  --set databaseHost=${DB_HOST} \\\\\\n  --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n  --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n  --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n  --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n  --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n  --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n  --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n  --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n  --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n  --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n  --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n  --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n  --set tracing.address=http://ingest.observability:4318 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n  --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n  --set provisionShardClusterLimit=100 \\\\\\n  --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n  --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n  --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n  --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n  --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n  --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n  --set azureOperatorsMI.roleSetName=dev \\\\\\n  --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036100790388527104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4507,"pr_state":"closed","pr_sha":"e955a59856d2a78adee812655213f7e96db841ee","final_merged_sha":"0e19405a0a977d3cd718004121a44142d3aebb42","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T15:18:41Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e5795d074c12add671edc71de84f79c8beaf42133289951a13990cc6214321c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036100790388527104","occurred_at":"2026-03-23T15:18:41Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036094471380144128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"9c5bf62b593d47ee992fff5d3e1cb249dde2bfc1","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T14:42:07Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6fad40a70e720845e2a3d70ca9f4895331a1555f4916f1abec4170a510ff7024","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036094471380144128","occurred_at":"2026-03-23T14:42:07Z","failed_at":"provision","signature_id":"a0c932ca4dc53ce932aff32aef5e2f1d467a6777304e91c599f0240413c8dd57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/kube-events","test_suite":"step graph","failure_text":"time=2026-03-23T15:33:21.717Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events description=\"Step kube-events\\n Kind: Helm\\n\"\ntime=2026-03-23T15:33:22.530Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events values=\"map[enabled:true image:map[digest:sha256:f907f0eb97bdf12502907c5003845276d41662736d7a5462ae597eb74e337bc9 registry:kubernetesshared.azurecr.io repository:shared/kube-events]]\"\ntime=2026-03-23T15:33:22.530Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-03-23T15:33:23.353Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace monitoring: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98d065c7e307c3a4cb1ea27a26d7c5068451c33863441ab95911619f03c87b93","failure_pattern":"failed to apply namespace monitoring: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace monitoring: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["98d065c7e307c3a4cb1ea27a26d7c5068451c33863441ab95911619f03c87b93"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036094338735280128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4510,"pr_state":"closed","pr_sha":"55f4344a90d7e862fceb906b74d95fae517ee7d9","final_merged_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-23T14:35:34Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4576/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036085854593617920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4576,"pr_state":"closed","pr_sha":"4eda0032750666275592116643efcf64bdf1b330","final_merged_sha":"4eda0032750666275592116643efcf64bdf1b330","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-23T14:21:08Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"52d98e65b0e8eb4913255828967668bdab3560fd815f060b536ba8cb3bde9cd5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4576/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036085854593617920","occurred_at":"2026-03-23T14:21:08Z","failed_at":"e2e","signature_id":"549363102869184e368ae7850f89249659db521c08afd9bde39c7c9aff865809","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f54b60\u003e: \n    failed to start credential request: POST https://rp.j3617920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-lqctzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://rp.j3617920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-lqctzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001561b30\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"307\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"437b836e-e757-4253-8dff-267c064c23e3\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 15:19:03 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000f54aa0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j3617920.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-lqctzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n                        ],\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T15:18:54Z\\\"}\",\n                        ],\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"Authorization\": [\"redacted\"],\n      ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f54b60\u003e: \n    failed to start credential request: POST https://rp.j3617920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-lqctzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://rp.j3617920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-lqctzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001561b30\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"307\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"437b836e-e757-4253-8dff-267c064c23e3\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 23 Mar 2026 15:19:03 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000f54aa0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"rp.j3617920.hcpsvc.osadev.cloud\",\n                        Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-lqctzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\",\n                        RawPath: \"\",\n                        OmitHost: false,\n                        ForceQuery: false,\n                        RawQuery: \"api-version=2024-06-10-preview\",\n                        Fragment: \"\",\n                        RawFragment: \"\",\n                    },\n                    Proto: \"HTTP/1.1\",\n                    ProtoMajor: 1,\n                    ProtoMinor: 1,\n                    Header: {\n                        \"User-Agent\": [\n                            \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n                        ],\n                        \"X-Ms-Arm-Resource-System-Data\": [\n                            \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-23T15:18:54Z\\\"}\",\n                        ],\n                        \"X-Ms-Identity-Url\": [\n                            \"https://dummyhost.identity.azure.net\",\n                        ],\n                        \"Authorization\": [\"redacted\"],\n      ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4581/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036080581212639232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4581,"pr_state":"closed","pr_sha":"eb2fe3a4f3987f6c2ba48f911a4b1dc731143b6e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-23T14:00:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036079917086543872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4580,"pr_state":"closed","pr_sha":"c26738b432de71922abb4a4c6e4169d6822e2588","final_merged_sha":"c26738b432de71922abb4a4c6e4169d6822e2588","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T13:57:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036076282042650624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4257,"pr_state":"closed","pr_sha":"06e9152e575fe1af81ab23347da2f44373ebcf48","final_merged_sha":"f9b0dd45c8c2499c52d47ec79f8297cdae4dfe1f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T13:43:08Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d0205657485d1e8ffcb4ae5aede19f63dae6ec55595d170e1088104bbf533eab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036076282042650624","occurred_at":"2026-03-23T13:43:08Z","failed_at":"e2e","signature_id":"ea504f45b2b9a7b1b815b3e8dbdc3c559e63e0d4be0f0fae751e12aad07f630c","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f91320\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-9trnk9\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-9trnk9, error: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-9trnk9\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-9trnk9, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000e729f0\u003e{\n                s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-9trnk9\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f91320\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-9trnk9\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-9trnk9, error: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-9trnk9\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-9trnk9, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000e729f0\u003e{\n                s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-9trnk9\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-9trnk9\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-9trnk9, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036072524021764096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"closed","pr_sha":"9d25e95d451b47f2e7b033c2904eb8d83195854b","final_merged_sha":"9feee09ab6996cfe7dd433b262345799b7851593","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-23T13:28:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4579/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036060644553265152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4579,"pr_state":"closed","pr_sha":"35bd2ace7e36ec862a86d56aaf86cdfd7bb32fba","final_merged_sha":"35bd2ace7e36ec862a86d56aaf86cdfd7bb32fba","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T12:41:00Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036054468801335296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4494,"pr_state":"closed","pr_sha":"b6d9c642ca814d40f9bb7032fbbf894bc4edfd69","final_merged_sha":"b6d9c642ca814d40f9bb7032fbbf894bc4edfd69","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T12:16:28Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4576/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036029699611693056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4576,"pr_state":"closed","pr_sha":"4eda0032750666275592116643efcf64bdf1b330","final_merged_sha":"4eda0032750666275592116643efcf64bdf1b330","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T10:38:02Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036024908042997760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4566,"pr_state":"open","pr_sha":"05658ee1a4f280929133182297024b849fd4837f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-23T10:18:58Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036021346776911872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4553,"pr_state":"closed","pr_sha":"7573bfbf6c5a2ea4b6ebbc79910eeabcb424ca64","final_merged_sha":"7573bfbf6c5a2ea4b6ebbc79910eeabcb424ca64","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T10:04:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036013563725746176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4494,"pr_state":"closed","pr_sha":"b6d9c642ca814d40f9bb7032fbbf894bc4edfd69","final_merged_sha":"b6d9c642ca814d40f9bb7032fbbf894bc4edfd69","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T09:33:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036007417749180416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4510,"pr_state":"closed","pr_sha":"704b8cdeaa0c942e9c4bedf387c789d55bbec4cd","final_merged_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-23T09:09:30Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036002581834108928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4575,"pr_state":"closed","pr_sha":"3ef943a923f87278ed4d8fb42032e6607c567f66","final_merged_sha":"3ef943a923f87278ed4d8fb42032e6607c567f66","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T08:50:17Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4569/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035987949048303616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4569,"pr_state":"closed","pr_sha":"9ce586d37fefdc23d08fd7b9f056ad8e5dc1bf51","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-23T07:52:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035987200453120000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4566,"pr_state":"open","pr_sha":"3db9304da5d28560e0c5551f1963c901f09f799e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-23T07:49:10Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035974611560042496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4575,"pr_state":"closed","pr_sha":"3ef943a923f87278ed4d8fb42032e6607c567f66","final_merged_sha":"3ef943a923f87278ed4d8fb42032e6607c567f66","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T06:59:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035966326404550656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4575,"pr_state":"closed","pr_sha":"3ef943a923f87278ed4d8fb42032e6607c567f66","final_merged_sha":"3ef943a923f87278ed4d8fb42032e6607c567f66","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-23T06:26:13Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"83ae7cf154f8ca9f3637541302a40213bd345d29500d1847d05851162b32467a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035966326404550656","occurred_at":"2026-03-23T06:26:13Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4574/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035940804861628416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4574,"pr_state":"closed","pr_sha":"fec4178ae99a6ed919d6646aec5fc62e26fef3ab","final_merged_sha":"162cc07eb137e2c6e706c814b9d10c4449bc72e0","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-23T04:44:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4574/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035917562188926976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4574,"pr_state":"closed","pr_sha":"36734643c812eeb03d695ae14bbdc20f1692f146","final_merged_sha":"162cc07eb137e2c6e706c814b9d10c4449bc72e0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T03:12:26Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1bff99d802a2f0a4cd20b0b0f1ab1dac3a84264bc7802e15bbd45625fdf0b736","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4574/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035917562188926976","occurred_at":"2026-03-23T03:12:26Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-23T03:31:28.344Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-23T03:31:30.065Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-23T03:31:31.832Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=06e0035c2cc05044b786bcb1bfbff81839e23ae5acbb189a590b7575bbc787f8\ntime=2026-03-23T03:36:03.164Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8926976-svc/providers/Microsoft.Resources/deployments/06e0035c2cc05044b786bcb1bfbff81839e23ae5acbb189a590b7575bbc787f8/operationStatuses/08584273701947108281\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"InvalidTemplateDeployment\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The template deployment 'cluster-5ynmvygxl445g' is not valid according to the validation procedure. The tracking id is '51d60b8b-f5d8-4d36-b610-bfb1ddf3c550'. See inner errors for details.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"Preflight validation check for resource(s) for container service prow-j8926976-svc in resource group hcp-underlay-prow-j8926976-svc failed. Message: Requested change in revisions is not allowed. Reason: Change from current revision(s) [] to requested [\\\\\\\\\\\\\\\"asm-1-26\\\\\\\\\\\\\\\" \\\\\\\\\\\\\\\"asm-1-28\\\\\\\\\\\\\\\"] is not valid. Please verify current revision is not empty and requested revision is returned in 'az aks mesh get-upgrades' output.. Details: \\\\\\\"\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"946ff1ea098b011b7a6621354ef6ebf0bc2042048a372308a2aec2fc554f0716","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message Preflight validation check for resource(s) for container service prow-j8926976-svc in resource group \u003cresource-group\u003e failed. Message: Requested change in revisions is not allowed. Reason: Change from current revision(s) [] to requested [\"asm-1-26\" \"asm-1-28","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["946ff1ea098b011b7a6621354ef6ebf0bc2042048a372308a2aec2fc554f0716"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4574/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035906412890034176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4574,"pr_state":"closed","pr_sha":"36734643c812eeb03d695ae14bbdc20f1692f146","final_merged_sha":"162cc07eb137e2c6e706c814b9d10c4449bc72e0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-23T02:28:08Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4515d41bad83a6068f6a4de738fa9e87c7076348fc5691e89d20be8afa71a36c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4574/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035906412890034176","occurred_at":"2026-03-23T02:28:08Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-23T02:50:25.703Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-23T02:50:27.607Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-23T02:50:29.102Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=4dcbe397d7cb9641f716e90b063fddfcc8005b59dc856eb846fa9e757aa8eeab\ntime=2026-03-23T02:54:30.280Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0034176-svc/providers/Microsoft.Resources/deployments/4dcbe397d7cb9641f716e90b063fddfcc8005b59dc856eb846fa9e757aa8eeab/operationStatuses/08584273726571660425\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"InvalidTemplateDeployment\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The template deployment 'cluster-egnrtjj2oqpg6' is not valid according to the validation procedure. The tracking id is '5b0f0640-ef01-494d-99f4-125dc24c7413'. See inner errors for details.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"Preflight validation check for resource(s) for container service prow-j0034176-svc in resource group hcp-underlay-prow-j0034176-svc failed. Message: Requested change in revisions is not allowed. Reason: Change from current revision(s) [] to requested [\\\\\\\\\\\\\\\"asm-1-26\\\\\\\\\\\\\\\" \\\\\\\\\\\\\\\"asm-1-28\\\\\\\\\\\\\\\"] is not valid. Please verify current revision is not empty and requested revision is returned in 'az aks mesh get-upgrades' output.. Details: \\\\\\\"\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1fa393df7969ec4d3758777910db22de474acebb0caeea8ca9d00f9a82177486","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message Preflight validation check for resource(s) for container service prow-j0034176-svc in resource group \u003cresource-group\u003e failed. Message: Requested change in revisions is not allowed. Reason: Change from current revision(s) [] to requested [\"asm-1-26\" \"asm-1-28","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1fa393df7969ec4d3758777910db22de474acebb0caeea8ca9d00f9a82177486"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035899662258409472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4463,"pr_state":"open","pr_sha":"8bf53059836d7c658117add712660a10ebbab48b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-23T02:01:14Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4570/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035876352707530752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4570,"pr_state":"closed","pr_sha":"6f2eb161f6d8a63a906c82e09f9d837e1e248b6a","final_merged_sha":"6f2eb161f6d8a63a906c82e09f9d837e1e248b6a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-23T00:28:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}