{"meta":{"date":"2026-07-23","timezone":"UTC","generated_at":"2026-08-27T11:18:12Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2080427937647038464","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-23T23:00:47Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-23T14:34:14Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"1013c4bd564ec15781be9a5fe98586912a0313566aa52f5bf824128b3dbcfcad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"6b64c0497cb6a7de0f6df1316cd7aea1515100901d5aeae62f2e291545f5df32","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c4c4e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-z6d2ccxkwbm9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc7945ac-71c6-41ff-8506-5f96ae9eb9aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc7945ac-71c6-41ff-8506-5f96ae9eb9aa\",\n      \"name\": \"dc7945ac-71c6-41ff-8506-5f96ae9eb9aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:56:53.295655313Z\",\n      \"endTime\": \"2026-07-23T15:16:00.70859883Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c4c4e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-z6d2ccxkwbm9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc7945ac-71c6-41ff-8506-5f96ae9eb9aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc7945ac-71c6-41ff-8506-5f96ae9eb9aa\",\n      \"name\": \"dc7945ac-71c6-41ff-8506-5f96ae9eb9aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:56:53.295655313Z\",\n      \"endTime\": \"2026-07-23T15:16:00.70859883Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1ed5b3863e6f0853dfb162669ddd03675389f2d423f1fa598b099c6f061926f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"ca2a57c00f153b6d7488071e8d97ffb596848ffc1e258c95a04400b39f327f42","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc5380\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-fmnzp4sdw2lh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d433c12f-48e3-4234-8059-98785a534ca4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d433c12f-48e3-4234-8059-98785a534ca4\",\n      \"name\": \"d433c12f-48e3-4234-8059-98785a534ca4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:56:00.123383638Z\",\n      \"endTime\": \"2026-07-23T15:15:00.488210427Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc5380\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-fmnzp4sdw2lh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d433c12f-48e3-4234-8059-98785a534ca4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d433c12f-48e3-4234-8059-98785a534ca4\",\n      \"name\": \"d433c12f-48e3-4234-8059-98785a534ca4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:56:00.123383638Z\",\n      \"endTime\": \"2026-07-23T15:15:00.488210427Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"312e371cad827158a3549e42ff005d7e6ce038c961683e69713f148a71c9858b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"4c0750c76c5e4a46d72be559656e9da35418529ee0242ad589653e7f2910b0cf","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c5d0a0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-kzmh2hsvhvk8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e6475fb8-7d66-4a3f-9a11-c8ce9a4ff903\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e6475fb8-7d66-4a3f-9a11-c8ce9a4ff903\",\n      \"name\": \"e6475fb8-7d66-4a3f-9a11-c8ce9a4ff903\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:38:51.52148988Z\",\n      \"endTime\": \"2026-07-23T15:57:51.229369914Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c5d0a0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-kzmh2hsvhvk8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e6475fb8-7d66-4a3f-9a11-c8ce9a4ff903\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e6475fb8-7d66-4a3f-9a11-c8ce9a4ff903\",\n      \"name\": \"e6475fb8-7d66-4a3f-9a11-c8ce9a4ff903\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:38:51.52148988Z\",\n      \"endTime\": \"2026-07-23T15:57:51.229369914Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4da4d969204171fdcbf04030d4b673bf1f0de46d9126098bb9a221fd59210915","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"5a371337c5131b79943f94870760e02310f49052df1b8843b9661a8c70afd761","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00107c1a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-k675rkhrbpwf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b7e2f89c-f64e-43f3-98b1-b2a73b98579c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b7e2f89c-f64e-43f3-98b1-b2a73b98579c\",\n      \"name\": \"b7e2f89c-f64e-43f3-98b1-b2a73b98579c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:58:12.217171113Z\",\n      \"endTime\": \"2026-07-23T15:17:30.750214168Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00107c1a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-k675rkhrbpwf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b7e2f89c-f64e-43f3-98b1-b2a73b98579c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b7e2f89c-f64e-43f3-98b1-b2a73b98579c\",\n      \"name\": \"b7e2f89c-f64e-43f3-98b1-b2a73b98579c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:58:12.217171113Z\",\n      \"endTime\": \"2026-07-23T15:17:30.750214168Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"59220c7b9a2ad081fe1c13417bc86ad1a4c4da3eef5679bad746e2a498ab6830","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"f1aebeeb7492bd08c71a6b12c63343c08386503b6840ef053c0c3d9ecad34f5d","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008f41c0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-d7jl5l9g8t4g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4fca8419-704b-41d1-b240-785c6b2babf8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4fca8419-704b-41d1-b240-785c6b2babf8\",\n      \"name\": \"4fca8419-704b-41d1-b240-785c6b2babf8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:56:20.757280221Z\",\n      \"endTime\": \"2026-07-23T15:15:30.708956769Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008f41c0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-d7jl5l9g8t4g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4fca8419-704b-41d1-b240-785c6b2babf8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4fca8419-704b-41d1-b240-785c6b2babf8\",\n      \"name\": \"4fca8419-704b-41d1-b240-785c6b2babf8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:56:20.757280221Z\",\n      \"endTime\": \"2026-07-23T15:15:30.708956769Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"66679b933efdfc213a6f87b5ea6d255c66e2d1f6e3f3490b663383a809be84d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"8a2bdaa74464be4ae4f7085471004a2892b6b516012862043fe6c37bb6ddae92","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-22-2hkmp8kgmll4/cluster-candidate-4-22-zqcp95 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c78320\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-zqcp95: failed waiting for cluster=\"cluster-candidate-4-22-zqcp95\" in resourcegroup=\"rg-candidate-4-22-2hkmp8kgmll4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7dd0acb6-1bcd-4cbd-a77e-1e2698a80ca7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7dd0acb6-1bcd-4cbd-a77e-1e2698a80ca7\",\n      \"name\": \"7dd0acb6-1bcd-4cbd-a77e-1e2698a80ca7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:57:03.082121392Z\",\n      \"endTime\": \"2026-07-23T15:16:10.601844479Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-22-2hkmp8kgmll4/cluster-candidate-4-22-zqcp95 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c78320\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-zqcp95: failed waiting for cluster=\"cluster-candidate-4-22-zqcp95\" in resourcegroup=\"rg-candidate-4-22-2hkmp8kgmll4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7dd0acb6-1bcd-4cbd-a77e-1e2698a80ca7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7dd0acb6-1bcd-4cbd-a77e-1e2698a80ca7\",\n      \"name\": \"7dd0acb6-1bcd-4cbd-a77e-1e2698a80ca7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:57:03.082121392Z\",\n      \"endTime\": \"2026-07-23T15:16:10.601844479Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"43b4c0ecc7cfb52979675ed0f02de9aacdd82a8bb5bb24f9b95172e131f6846b","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"79184c781c756b401721eeae5b2c1d5c875d6b67fe69f5c6d2988b6f75f69b76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"fc915ba250e390d158815e863f80c25f3e4cbc8021425841dcebdf7862c3f2e7","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f326f0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-qp8jkwnz8g6c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qp8jkwnz8g6c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f326f0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-qp8jkwnz8g6c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qp8jkwnz8g6c, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-qp8jkwnz8g6c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qp8jkwnz8g6c, error: context deadline exceeded"}},{"row_id":"8c38e080eede2b61049598d54c9fbc00e88e4c5b15d49be68f0a18849789ea03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"0909626138a55ab7c8dbd794c8b3a77fafa7aac5f3f2c44edb62238255966d8a","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00045ef40\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-btrlf49mx4mr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f8bb69e-3dbc-46bc-b77f-ac4c9b9f8f96\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f8bb69e-3dbc-46bc-b77f-ac4c9b9f8f96\",\n      \"name\": \"5f8bb69e-3dbc-46bc-b77f-ac4c9b9f8f96\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:29:57.199203941Z\",\n      \"endTime\": \"2026-07-23T15:49:11.470560955Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00045ef40\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-btrlf49mx4mr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f8bb69e-3dbc-46bc-b77f-ac4c9b9f8f96\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f8bb69e-3dbc-46bc-b77f-ac4c9b9f8f96\",\n      \"name\": \"5f8bb69e-3dbc-46bc-b77f-ac4c9b9f8f96\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:29:57.199203941Z\",\n      \"endTime\": \"2026-07-23T15:49:11.470560955Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a485027820f9e3ff218879471ff567a17eafdab1328030598c27cf47ea26a5a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"d712d5ce17264f0ef550496665e2a7bd7d67dfac479ff00deed5bcb5e610f7c9","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"a58692a72826ac64933766c6f9bb0284eb762500cb793e79157761abfe73897c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"2fc2833dd7943e0267fbd875c3cb47629186c64ce8a679a0b0e9dc7f1789a8d4","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-kr2pqb\" with version 4.20.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e02700\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-kr2pqb: failed waiting for cluster=\"cluster-zstream-4-20-kr2pqb\" in resourcegroup=\"rg-zstream-upgrade-4-20-5gp2mtdbkppf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1b92cfe-14c4-4841-a691-dcb15fd4bada\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1b92cfe-14c4-4841-a691-dcb15fd4bada\",\n      \"name\": \"a1b92cfe-14c4-4841-a691-dcb15fd4bada\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:33:05.814258572Z\",\n      \"endTime\": \"2026-07-23T15:52:11.319531705Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-kr2pqb\" with version 4.20.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e02700\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-kr2pqb: failed waiting for cluster=\"cluster-zstream-4-20-kr2pqb\" in resourcegroup=\"rg-zstream-upgrade-4-20-5gp2mtdbkppf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1b92cfe-14c4-4841-a691-dcb15fd4bada\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1b92cfe-14c4-4841-a691-dcb15fd4bada\",\n      \"name\": \"a1b92cfe-14c4-4841-a691-dcb15fd4bada\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:33:05.814258572Z\",\n      \"endTime\": \"2026-07-23T15:52:11.319531705Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b7aadd9edbb0042fc804778ce3df9db86d29837e569f4be000cf24370d726151","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"fe4559ffec4f3ebb39829cc2d25299d572c52b91485aafaa31fc6c7b5d86ebf1","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001012a40\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-vtmwlt59mcqb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98e7c2b5-126e-4d99-a480-38555e004a37\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98e7c2b5-126e-4d99-a480-38555e004a37\",\n      \"name\": \"98e7c2b5-126e-4d99-a480-38555e004a37\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:32:15.307228267Z\",\n      \"endTime\": \"2026-07-23T15:51:31.367394587Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001012a40\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-vtmwlt59mcqb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98e7c2b5-126e-4d99-a480-38555e004a37\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98e7c2b5-126e-4d99-a480-38555e004a37\",\n      \"name\": \"98e7c2b5-126e-4d99-a480-38555e004a37\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:32:15.307228267Z\",\n      \"endTime\": \"2026-07-23T15:51:31.367394587Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cf3894a1f5b1e4f1cd75c86da3f5e3301e437bde44f2e3994e4c6a70f4edea60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"bb17f5b2b19e58bc0ec041403d55521015c5e5fb9b4d2b13c8c0c5fe246700a4","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00115c360\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hsklklhdjwx5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d324661b-7e24-4358-8d2f-caad268deb7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d324661b-7e24-4358-8d2f-caad268deb7d\",\n      \"name\": \"d324661b-7e24-4358-8d2f-caad268deb7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:58:54.105750542Z\",\n      \"endTime\": \"2026-07-23T15:18:00.560174863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00115c360\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hsklklhdjwx5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d324661b-7e24-4358-8d2f-caad268deb7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d324661b-7e24-4358-8d2f-caad268deb7d\",\n      \"name\": \"d324661b-7e24-4358-8d2f-caad268deb7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:58:54.105750542Z\",\n      \"endTime\": \"2026-07-23T15:18:00.560174863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d8e09c821c952486424e53bafedba912a1378141f3409984c416f11bbc093701","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","failed_at":"e2e","signature_id":"77259c9c04a7ea64f63b5aa40837444b03f3056038b0d685ff4e584482f0e763","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cec5c0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-hq8vhpn7hs7z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13ec6fc7-0fe9-440e-b954-d90bbd276f01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13ec6fc7-0fe9-440e-b954-d90bbd276f01\",\n      \"name\": \"13ec6fc7-0fe9-440e-b954-d90bbd276f01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:56:31.891603533Z\",\n      \"endTime\": \"2026-07-23T15:15:50.558670399Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cec5c0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-hq8vhpn7hs7z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13ec6fc7-0fe9-440e-b954-d90bbd276f01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13ec6fc7-0fe9-440e-b954-d90bbd276f01\",\n      \"name\": \"13ec6fc7-0fe9-440e-b954-d90bbd276f01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:56:31.891603533Z\",\n      \"endTime\": \"2026-07-23T15:15:50.558670399Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","43b4c0ecc7cfb52979675ed0f02de9aacdd82a8bb5bb24f9b95172e131f6846b","4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080143850206138368","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-23T04:11:53Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a1bf2dc72bdf627c4489e7c4b1a13f5877497497c7e52a5231b6535270737484","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080143850206138368","occurred_at":"2026-07-23T04:11:53Z","failed_at":"e2e","signature_id":"ff776f26d065fc5418b757e9c6b4187c71a02c6c48dc47fb83ce32ca7f8ef27a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:181]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-zxpbdh\" with install version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cf11c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-zxpbdh: failed waiting for cluster=\"cp-ystream-upgrade-4-21-zxpbdh\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-4s5rv26vwmxd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/73d26d21-c65b-4a19-9eb0-29e0ae5aacee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/73d26d21-c65b-4a19-9eb0-29e0ae5aacee\",\n      \"name\": \"73d26d21-c65b-4a19-9eb0-29e0ae5aacee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T05:03:31.056503296Z\",\n      \"endTime\": \"2026-07-23T05:22:34.512235877Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:181]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-zxpbdh\" with install version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cf11c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-zxpbdh: failed waiting for cluster=\"cp-ystream-upgrade-4-21-zxpbdh\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-4s5rv26vwmxd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/73d26d21-c65b-4a19-9eb0-29e0ae5aacee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/73d26d21-c65b-4a19-9eb0-29e0ae5aacee\",\n      \"name\": \"73d26d21-c65b-4a19-9eb0-29e0ae5aacee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T05:03:31.056503296Z\",\n      \"endTime\": \"2026-07-23T05:22:34.512235877Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2080110844472987648","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-23T02:00:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}