{"meta":{"start_date":"2026-08-15","end_date":"2026-08-15","start_at":"2026-08-15T00:00:00Z","end_at":"2026-08-16T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T00:47:19Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":95,"matched_occurrences":95,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"46e30b272cd4ab3ae13194cd0f744e27ec0c9420bc1323c7d2de64c29ecb7c49","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, community-operators-catalog, redhat-marketplace-catalog, redhat-operators-catalog; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":50,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":50,"after_last_push_count":235,"trend_counts":[0,0,0,0,92,93,50],"trend_range":"2026-08-09..2026-08-15 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-hqnc9d\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001612e20\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-hqnc9d: failed waiting for cluster=\"cluster-zstream-4-21-hqnc9d\" in resourcegroup=\"rg-zstream-upgrade-4-21-mwgmgvhm9c22\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5bb69322-8f88-46e2-8a34-295b0972f2cb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5bb69322-8f88-46e2-8a34-295b0972f2cb\",\n      \"name\": \"5bb69322-8f88-46e2-8a34-295b0972f2cb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:35:18.719742409Z\",\n      \"endTime\": \"2026-08-15T23:54:26.184039428Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, redhat-operators-catalog, certified-operators-catalog, redhat-marketplace-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-hqnc9d\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001612e20\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-hqnc9d: failed waiting for cluster=\"cluster-zstream-4-21-hqnc9d\" in resourcegroup=\"rg-zstream-upgrade-4-21-mwgmgvhm9c22\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5bb69322-8f88-46e2-8a34-295b0972f2cb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5bb69322-8f88-46e2-8a34-295b0972f2cb\",\n      \"name\": \"5bb69322-8f88-46e2-8a34-295b0972f2cb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:35:18.719742409Z\",\n      \"endTime\": \"2026-08-15T23:54:26.184039428Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, redhat-operators-catalog, certified-operators-catalog, redhat-marketplace-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-79lngg\" with version 4.22.8 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c95200\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-79lngg: failed waiting for cluster=\"cluster-zstream-4-22-79lngg\" in resourcegroup=\"rg-zstream-upgrade-4-22-nqf5krvgp245\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/353795d0-f8ad-4f1f-9ce7-05d7bd6012a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/353795d0-f8ad-4f1f-9ce7-05d7bd6012a3\",\n      \"name\": \"353795d0-f8ad-4f1f-9ce7-05d7bd6012a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:36:13.973377342Z\",\n      \"endTime\": \"2026-08-15T23:55:16.079782692Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, redhat-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-79lngg\" with version 4.22.8 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c95200\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-79lngg: failed waiting for cluster=\"cluster-zstream-4-22-79lngg\" in resourcegroup=\"rg-zstream-upgrade-4-22-nqf5krvgp245\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/353795d0-f8ad-4f1f-9ce7-05d7bd6012a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/353795d0-f8ad-4f1f-9ce7-05d7bd6012a3\",\n      \"name\": \"353795d0-f8ad-4f1f-9ce7-05d7bd6012a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:36:13.973377342Z\",\n      \"endTime\": \"2026-08-15T23:55:16.079782692Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, redhat-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-0-jh9jkgmf6s5w/cluster-nightly-5-0-pw68ph should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a86e00\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-pw68ph: failed waiting for cluster=\"cluster-nightly-5-0-pw68ph\" in resourcegroup=\"rg-nightly-5-0-jh9jkgmf6s5w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2dbf3b19-8909-482d-bce8-e30234b8d35b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2dbf3b19-8909-482d-bce8-e30234b8d35b\",\n      \"name\": \"2dbf3b19-8909-482d-bce8-e30234b8d35b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:35:17.022024299Z\",\n      \"endTime\": \"2026-08-15T23:54:26.142792631Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, redhat-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-0-jh9jkgmf6s5w/cluster-nightly-5-0-pw68ph should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a86e00\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-pw68ph: failed waiting for cluster=\"cluster-nightly-5-0-pw68ph\" in resourcegroup=\"rg-nightly-5-0-jh9jkgmf6s5w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2dbf3b19-8909-482d-bce8-e30234b8d35b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2dbf3b19-8909-482d-bce8-e30234b8d35b\",\n      \"name\": \"2dbf3b19-8909-482d-bce8-e30234b8d35b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:35:17.022024299Z\",\n      \"endTime\": \"2026-08-15T23:54:26.142792631Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, redhat-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"3be7a5c31b6a583283c6cf3f2dc5ef47bf518ddb8d850f42c20c269797374ce0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"91e6571c2b47c72de331c16ea6e0d1568e4128d9409afc2dbbf35a45a46192bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"c518877c789361b6585e7f229b24f420272eadd92a622bf0ff60dfd9dbc6e781","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"eafc5dda97b020f4ac13fd59c66978402d9971a7e11a1d575bc6fd0931934798","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"0422dd6a5a7892ab1bb9f84cd67aba0c8b02a2d66965a709d7785a74a2235d14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"0b1320c721eb397741978e9e779babd8653af0b04e95874bfc9df865c1f04085","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"1553facf294d63041543947979eca957baf38a72c37ad15895fd72ebcb2badee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"207fba2e1e9a1c05cac1d832d50bd34c1a0c28921e289d3ad2ac0de0dc2122af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"25fcb9567a865ace4afc59c7e5519ce2b71c698fd8f6406f50edc6ea85b3c8f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"2ce6601a1e30fd08ed406d0ee5af571d13d577aab15c82565db2acf943f1e5db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"2f05afff3d8daedba7690978856ae4ff399c9e3ce38992bec10dd973fa0fff27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"344e1e0a0cfb86aea8f80f7d8eab9d5d1f079854812d05abca46ac8ec3310ea1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"394d0806ae99b3746e84ce075a24d21a995c96bbff880b63c8266f06ec35d2df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"3f95846c099b1c5303711775ee8fb1111a0fb072dcc94915a2a82bd90e45558f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"4985ebf182bbeb1805c6c10f61ddc670b06e3e0cf3627c7ed71f8ade0b415db9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"55af77283e4053c6fad06c42bb583c5ea1eb52c7abbf37d03a1de3a96dc8ac44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"5e7740a7008accca7fcb3aae01a23b811382b7dc90966ee3f1b9d8a50c34d2cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"63e2d7ee594bb6e8e8b1233586f9218dd3ebcc9eee3cf8eb15a1da70ac04d97f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"6515790a790cd8eb5aa0198e7141e2715ad3255b28bc8d4f0df74eeeb6fdb8ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"65280be20b0dbe796146b55ed6db2e30915b3862ea10259aff718ca3bb78f1de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"6f3e0c4069b19cc27358d80092e6844b73e2fcc25862ea20d01d72943432290b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"7218ea1053e3165bb192c5768d10b7baf692a0d0ae7ad1982c2dd85bb9451ed0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"7458a2eaee6f1613a73a144fd6d517286c9bbb06ae9b62335ae8d4507850a9a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"761f300914ed4ac724332110aee0549131300b9d2fa547d3c477cb0c37f71e2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"879960ea316bd50f1e3403f25db2093c2ecb33bcc223babc2d3a26cb220824ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"8bfc0766d2c547b095585f55d5c6ff1fdb99663255b9f0034f72a4261a1222a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"8d5465f95ef563aea512cdf693e286aea7f49d6e5f8908ac69324cb988aa1bc0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"8f798f09ba976c3e5b98f1f957aa60ca08867a840b5cc6aab1ba3752d70b37d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"92b068e255c6698bfafd0efe4b51508fe490c031365896c7b4890c530aef8f8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"9731957c1d1caadcb8a6060c5e81a09f057aad3e7d97e4f564b0bda35d8100d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"9cc805de4c5853a73881536df3acf6c8b72973a368d279129b18d0fd8cf03709","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"a6a5c70963c25431020d98beb56fea5e8ad95d0eba0f80a3b77c97964dc6d9ba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"ab2841dc1460ea80e4c26f5fcba0ce8c8f3386006a91aa41b158aba745e26a3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"b0170878fac7967b93f4572264478d81b15fc8d10b72aa8b1bea09141aa60c0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"b5df55d03930b7fdbd23fd999a792c5dff1abc7a21af5b912d5be4bb58124bb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"bfcbc0b24a2587752caa69d51d74cc8e609b177fef59333de3586a521f20b3ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"c43af5be51ae6e0723383fe5508832e6938885fdd7c3e8191bc38303dd150f89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"c57a55da8bca09ad7fee7a7082e523aa457be8de72d5c443e5e1995de2d3923f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"ca7d641feceeecf6ceae28875ce35ede0109ae8674dea6f2ce8651ef4ea08eae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"d5a04c46d552a13276f8564e4f16ca8bdb45276685515636b645af21b7add870","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"db498d3970b9a63fab8534328108c9f859531c84a8e830a33ce46f19f05256c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"db7de55ff81947567567cc80e829e5660aa9d109af57004fb279724e4db6036e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"e157db6cffa39f99e751f1a6c3026a0e83b7ef6570114db7da7f037e1ced5978","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"e3d20ab1ed9d86258c701c3660538cd62cb9c61ab78f656f39d22a6220e30d48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"ed4d9738ab10026981b57aed13cb126c4dff5da1c79bc1c8d1026729c0ea1b2e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"f2d380e7eb917646e8a85c88aa49e3d923c5b31efd9417ea9184a94c00b9c447","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"f4a27111bc5bc6884d2432404e0135e32006f757f7e680c0bda08d48d97c9210","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"f9b8df3acf10970badab68e4c5a7e3aeafb4caa0b32009a7d83799da2cab6c60","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"fca5e7eb7cf5d7bf42991143daeb4c8e89197c6831dee304af4b2bfd304d750e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"fd0569cf76c52cb8e3d25096781ab3c09eecd0c353e3d358ff451ce96e52abea","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":34,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":34,"after_last_push_count":107,"trend_counts":[0,0,0,0,34,34,34],"trend_range":"2026-08-09..2026-08-15 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-03"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-07T02:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd1720\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-mkh4qddtcnlv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/366d1c5a-e518-46cd-b8db-168810fa60bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/366d1c5a-e518-46cd-b8db-168810fa60bc\",\n      \"name\": \"366d1c5a-e518-46cd-b8db-168810fa60bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:36:36.131900873Z\",\n      \"endTime\": \"2026-08-15T23:55:36.645922521Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd1720\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-mkh4qddtcnlv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/366d1c5a-e518-46cd-b8db-168810fa60bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/366d1c5a-e518-46cd-b8db-168810fa60bc\",\n      \"name\": \"366d1c5a-e518-46cd-b8db-168810fa60bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:36:36.131900873Z\",\n      \"endTime\": \"2026-08-15T23:55:36.645922521Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eee9a0\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-hbwccm8jdjth\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dd527248-6026-427f-ae39-6bc0398b818a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dd527248-6026-427f-ae39-6bc0398b818a\",\n      \"name\": \"dd527248-6026-427f-ae39-6bc0398b818a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:36:44.351982807Z\",\n      \"endTime\": \"2026-08-15T23:55:46.579172195Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eee9a0\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-hbwccm8jdjth\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dd527248-6026-427f-ae39-6bc0398b818a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dd527248-6026-427f-ae39-6bc0398b818a\",\n      \"name\": \"dd527248-6026-427f-ae39-6bc0398b818a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:36:44.351982807Z\",\n      \"endTime\": \"2026-08-15T23:55:46.579172195Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f04ee0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-6md7nlk4bvc8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2cac58b-42cb-45b7-94fa-c6ab115a5751\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2cac58b-42cb-45b7-94fa-c6ab115a5751\",\n      \"name\": \"e2cac58b-42cb-45b7-94fa-c6ab115a5751\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:39:13.122579268Z\",\n      \"endTime\": \"2026-08-15T23:58:16.655159324Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f04ee0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-6md7nlk4bvc8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2cac58b-42cb-45b7-94fa-c6ab115a5751\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2cac58b-42cb-45b7-94fa-c6ab115a5751\",\n      \"name\": \"e2cac58b-42cb-45b7-94fa-c6ab115a5751\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:39:13.122579268Z\",\n      \"endTime\": \"2026-08-15T23:58:16.655159324Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"02a6fab4665c3a6da4ad64e1bb17706402121daf1e5ff957238f00caff1ee613","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"04b968c6bf37572a51cdf62a07f1182c44b2d1a44b3bdafcc3a6f22a6d0128dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"183c672856a01fcafc7435be6af252748fce2fb79f027a3b717284646a352ded","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"1ba344757ec3a37918433d1d6d543b5269a6aafa2972b7aa7611a2e8f0675f5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"1f1a84f8378eaa65853b8d825fc0bbfbb717d640aa40cfd5075f074ffb2e8a4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"1fc039636ac1dddc55304f221ad84f7a82115064cfbc2f6b1b38b4e400f339c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"242d88ccd8ebce9348b633eeeade90650a3f0a3ec251d620f762bd5b17c9891b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"2ebe0c96eef8d8fb2e5dd1e06cd7c4450ce87d5e48a88f30c288e26c8e0d4ed2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"2fb464e1d3ae9181fa6bf2da0d7e97ea982c676da5929373f961416fd75047aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"34d056431cedca66883784a55448fe14916617d05b3ea165b42f6362891ec6fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"370f36b1e14ebe8f06dbbbc70dee00b33e16f2dc8e87ed6f1f6444f20ebb6b07","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"4298cb8bfcc8fb72737077382b98df6721eefe5660dc84186dc9d5c8bc943491","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"4479569fa94de8c8ae3294ccb65a5ca17685f9ea460b90a52c722d61c7935805","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"55fb5eab6830b7e130c1ce80514c53e8719af2a4638ce6bafb86faa11f913760","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"687ab9706fb7d5aaf5f0afd0455089776edb44b691a4e201d04a8517fb5b71cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"6a6f256adb632fcf1b35a10a79d124e6ae31c8f6afcbba9cfeeb20efe8b6acae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"6d9a6b5be74420420d97012f4c1cc413a8b5176c532fcb0fc7927dbfb27d372e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"6da1acd515e8a642097f5e5fb2bdd202f53bc1d8d13fd67443977488538f0090","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"7566af70d7d632f9a8590228811d66126bed93a44aa3fee386c7095778f0fa27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"88effdf670ff94b4e68d03fc0aa82e2ee82c50b8de2236f24d8ad18e069ae459","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"8a2d3cada3645c817269aff85c39d1d41bb9ea8c06cc051c075df56b501439d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"8a31faba5a8673520aa147fe87db6ba42ff37676f3785f42fb7f30b580859ceb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"8df6206baab8c4899424ee1645998e4c31dffe811a9cbb906f42209c5699185c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"8ef6aae400f7a3904389c0c284e605aefc649ca48b1b259568363b6e55073495","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"907e0b7e88d609f3d15b64257c31d973ceaf8d049af05e4cdff9ab042fbb4096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"94a0d5f59b7b396c980a33414d70c57a3c888b1a61138bc397baabf24044eb75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"9a28bb843b37655a8b5a31e2f3bebe41066b5221826ea469626d16ec85e40756","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"a6e5713dbc21ace73b4f4fee7f90abedfb867e45ca3192d1842bf965c85b11d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"b79c46615d9c8f654715c63cdd64c01d484e49f633a35d790cb4ef8f45f41f0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"b9cf5443534208c6ad15713b1737db46c81aeacc63f82342963c6f362acc2191","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"bfec4f6965d8e894250e5884d2f7711fce998e3a796148c33a70a5f92deaab0b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"cc04f9b4f98a36e2a14f79e0a19af9833e687d956310fb4a0ba32440028db4b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"e69a5d4e2cd241699ed0f7ee884c52fd0ba1a99966478b6c3323cf15fe0d13d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"fbd6dda948a851a19466a2328cf363df874e711a19ee892cd0338b14cd57b4de","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"44e2d333ce62abafbaf16d046b61703140a463730eb39f1d9ff95ea9875c0757","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":11,"trend_counts":[0,0,0,2,3,3,2],"trend_range":"2026-08-09..2026-08-15 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-27"],"prior_runs_affected":11,"prior_last_seen_at":"2026-07-29T02:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":12,"trend_counts":[0,0,0,2,3,3,2],"trend_range":"2026-08-09..2026-08-15 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-20","2026-07-27"],"prior_runs_affected":12,"prior_last_seen_at":"2026-08-01T23:00:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.006s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.006s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"2cfe09be092d341b2785b1ec7877c8436ce5f3a66545f57a16f2cc75e59ab47e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-1-k44kkc, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-k44kkc in resource group rg-nightly-5-1-rxc7nshqpwvf, error: failed waiting for cluster=\"cluster-nightly-5-1-k44kkc\" in resourcegroup=\"rg-nightly-5-1-rxc7nshqpwvf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-k44kkc in resource group rg-nightly-5-1-rxc7nshqpwvf, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-09..2026-08-15 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-13","2026-07-20","2026-08-03"],"prior_runs_affected":22,"prior_last_seen_at":"2026-08-09T23:00:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00184e450\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-l2gj5qbx747t, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-l2gj5qbx747t\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-l2gj5qbx747t, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00184e450\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-l2gj5qbx747t, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-l2gj5qbx747t\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-l2gj5qbx747t, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-rxc7nshqpwvf/cluster-nightly-5-1-k44kkc should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011390b0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-k44kkc, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-k44kkc in resource group rg-nightly-5-1-rxc7nshqpwvf, error: failed waiting for cluster=\"cluster-nightly-5-1-k44kkc\" in resourcegroup=\"rg-nightly-5-1-rxc7nshqpwvf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-k44kkc in resource group rg-nightly-5-1-rxc7nshqpwvf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-rxc7nshqpwvf/cluster-nightly-5-1-k44kkc should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011390b0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-k44kkc, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-k44kkc in resource group rg-nightly-5-1-rxc7nshqpwvf, error: failed waiting for cluster=\"cluster-nightly-5-1-k44kkc\" in resourcegroup=\"rg-nightly-5-1-rxc7nshqpwvf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-k44kkc in resource group rg-nightly-5-1-rxc7nshqpwvf, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"65718f57577b56596b45d570f5cff984a09d1080e44a83e1692683b6382ffc0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"b907581464649121b94083fcb09df150e5b0736407fd54ebcbbac1c20aec344d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"077f77b9825f03821964d99637a7414de7ca52612e9a64bf240378e696ca5436","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, community-operators-catalog, redhat-operators-catalog; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,1,1],"trend_range":"2026-08-09..2026-08-15 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-23-l8h9jnppm5wd/cluster-nightly-4-23-zvhvqz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e809a0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-zvhvqz: failed waiting for cluster=\"cluster-nightly-4-23-zvhvqz\" in resourcegroup=\"rg-nightly-4-23-l8h9jnppm5wd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8cd4b8a0-1c93-4dec-9d60-226592ea3ad1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8cd4b8a0-1c93-4dec-9d60-226592ea3ad1\",\n      \"name\": \"8cd4b8a0-1c93-4dec-9d60-226592ea3ad1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:34:57.06470798Z\",\n      \"endTime\": \"2026-08-15T23:54:06.046848773Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, certified-operators-catalog, redhat-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-23-l8h9jnppm5wd/cluster-nightly-4-23-zvhvqz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e809a0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-zvhvqz: failed waiting for cluster=\"cluster-nightly-4-23-zvhvqz\" in resourcegroup=\"rg-nightly-4-23-l8h9jnppm5wd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8cd4b8a0-1c93-4dec-9d60-226592ea3ad1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8cd4b8a0-1c93-4dec-9d60-226592ea3ad1\",\n      \"name\": \"8cd4b8a0-1c93-4dec-9d60-226592ea3ad1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-15T23:34:57.06470798Z\",\n      \"endTime\": \"2026-08-15T23:54:06.046848773Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, certified-operators-catalog, redhat-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"7ce05954de04f747822c0d8f1796bb0c563a9c4ba2f926d6e42d35d0561e7bb1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"da65f908f97313ddbcc1cd3d11e952371e136eb64bf37368bcd0011d3f5a924e","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from \u003curl\u003e Get \"\u003curl\u003e\": context deadline exceeded","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.3.179:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.3.179:443/apis/packages.operators.coreos.com/v1\": context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-09..2026-08-15 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: failed to verify HCP cluster \"cluster-zstream-4-19-mtqb7z\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a6e000\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.3.179:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.3.179:443/apis/packages.operators.coreos.com/v1\": context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: failed to verify HCP cluster \"cluster-zstream-4-19-mtqb7z\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a6e000\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.3.179:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.3.179:443/apis/packages.operators.coreos.com/v1\": context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"12d313bfad10f8247708bf61a8688a1ee3b07de0dd453f2e47b3afd0f2ca9e65","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"31598de3d08c9110174e20603912e8c0af3d83fd5920aa7761ad7b8dcb25888b","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from \u003curl\u003e Get \"\u003curl\u003e\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.1.250:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.1.250:443/apis/packages.operators.coreos.com/v1\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-09..2026-08-15 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: failed to verify HCP cluster \"cluster-zstream-4-19-rjdk7n\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012612a8\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.1.250:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.1.250:443/apis/packages.operators.coreos.com/v1\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: failed to verify HCP cluster \"cluster-zstream-4-19-rjdk7n\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012612a8\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.1.250:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.1.250:443/apis/packages.operators.coreos.com/v1\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"2c72c0b690e763b46862e6cbaf5db1ad81078b478157d62aedd51e1d6864c1ae","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ab9e82a8af804bbb7c138751caa26c2cb132cf89c97d2f090acc75dc9a8b2e15","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from \u003curl\u003e Get \"\u003curl\u003e\": http: server gave HTTP response to HTTPS client","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.51.81:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.51.81:443/apis/packages.operators.coreos.com/v1\": http: server gave HTTP response to HTTPS client","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-09..2026-08-15 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:163]: failed to verify HCP cluster viability for back-level version 4.19\nUnexpected error:\n    \u003c*errors.joinError | 0xc001513e00\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.51.81:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.51.81:443/apis/packages.operators.coreos.com/v1\": http: server gave HTTP response to HTTPS client\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:163]: failed to verify HCP cluster viability for back-level version 4.19\nUnexpected error:\n    \u003c*errors.joinError | 0xc001513e00\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.51.81:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.51.81:443/apis/packages.operators.coreos.com/v1\": http: server gave HTTP response to HTTPS client\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2088762881725173760","occurred_at":"2026-08-15T23:00:52Z","signature_id":"cfb33fb2ab672290af54cd16dd68a5ab52ca7ce422043029580ebba6f5bf87b8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6477f6d87ddc79ca469b892f0755a20abbcbf9857b736992068b4e3c57909813","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from \u003curl\u003e Get \"\u003curl\u003e\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.205.164:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.205.164:443/apis/packages.operators.coreos.com/v1\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-08-09..2026-08-15 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:163]: failed to verify HCP cluster viability for back-level version 4.19\nUnexpected error:\n    \u003c*errors.joinError | 0xc000fb1860\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.205.164:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.205.164:443/apis/packages.operators.coreos.com/v1\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:163]: failed to verify HCP cluster viability for back-level version 4.19\nUnexpected error:\n    \u003c*errors.joinError | 0xc000fb1860\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.205.164:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.205.164:443/apis/packages.operators.coreos.com/v1\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088445790086238208","occurred_at":"2026-08-15T02:00:52Z","signature_id":"5e1b1a3a1dd0b2c2a3cb16f73a3d9f264ce1cb4b115848dd47b21e50ebfd7113","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}