{"meta":{"start_date":"2026-03-31","end_date":"2026-03-31","start_at":"2026-03-31T00:00:00Z","end_at":"2026-04-01T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T15:32:37Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":3,"failed_runs":2,"raw_occurrences":5,"matched_occurrences":5,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-qdzg54\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-03-25..2026-03-31 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":14,"prior_last_seen_at":"2026-03-26T19:56:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000916240\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pgdlrr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-pgdlrr, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-pgdlrr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-pgdlrr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010acbc0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-pgdlrr\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000916240\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pgdlrr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-pgdlrr, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-pgdlrr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-pgdlrr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010acbc0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-pgdlrr\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:136]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001750cf0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-qdzg54\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-qdzg54\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000fe0f60\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001750cc0\u003e{\n                msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-qdzg54\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000fe0f60\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:136]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001750cf0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-qdzg54\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-qdzg54\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000fe0f60\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001750cc0\u003e{\n                msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-qdzg54\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000fe0f60\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ab8570\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vkzvvw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-vkzvvw, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-vkzvvw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-vkzvvw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010f6130\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-vkzvvw\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ab8570\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vkzvvw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-vkzvvw, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-vkzvvw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-vkzvvw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010f6130\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-vkzvvw\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038973601599721472","occurred_at":"2026-03-31T13:36:01Z","signature_id":"9382cb9eb79eae42cc5dace1ccccc98e82e38345c45f04052607b8bfad63da73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038798596739960832","occurred_at":"2026-03-31T02:00:39Z","signature_id":"347defe85f9f6eb0c1f25c4ce72b2ff46edca527ac9746d258764abc2fb5baf2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038798596739960832","occurred_at":"2026-03-31T02:00:39Z","signature_id":"7d98ec92bd51c96dc08930d9f98e3246a9a1e48679cd5d21d4b53730fe1fa926","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038798596739960832","occurred_at":"2026-03-31T02:00:39Z","signature_id":"c98f63922f49eccb84d159af311d73ccaa7e94b2fcfdca094ca21546834d5777","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"953e20e9d1db6d0f9c7e6c5ea18e02689f21a0690a145ecf8a28e443cb3a9c26","failure_pattern":"add password: error status code received from the API","search_query":"add password: error status code received from the API","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-03-25..2026-03-31 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:252]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000517600\u003e: \n    add password: error status code received from the API\n    {\n        msg: \"add password: error status code received from the API\",\n        err: \u003c*odataerrors.ODataError | 0xc000b0ba40\u003e{\n            ApiError: {\n                Message: \"\",\n                ResponseStatusCode: 404,\n                ResponseHeaders: {\n                    header: {\n                        headers: {\n                            \"client-request-id\": {\n                                \"0c31e575-0500-4a44-aebc-155ee4c1eea2\": {},\n                            },\n                            \"content-type\": {\n                                \"application/json\": {},\n                            },\n                            \"x-ms-resource-unit\": {\"1\": {}},\n                            \"date\": {\n                                \"Tue, 31 Mar 2026 02:30:32 GMT\": {},\n                            },\n                            \"strict-transport-security\": {\n                                \"max-age=31536000\": {},\n                            },\n                            \"request-id\": {\n                                \"0c31e575-0500-4a44-aebc-155ee4c1eea2\": {},\n                            },\n                            \"x-ms-ags-diagnostic\": {\n                                \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"Canada Central\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"3\\\",\\\"ScaleUnit\\\":\\\"001\\\",\\\"RoleInstance\\\":\\\"YT1PEPF00003E4B\\\"}}\": {},\n                            },\n                            \"cache-control\": {\"no-cache\": {}},\n                            \"vary\": {\n                                \"Accept-Encoding\": {},\n                            },\n                        },\n                    },\n                },\n            },\n            backingStore: \u003c*store.InMemoryBackingStore | 0xc000517540\u003e{\n                returnOnlyChangedValues: false,\n                initializationCompleted: false,\n                store: {\n                    \"errorEscaped\": \u003c*odataerrors.MainError | 0xc000741cd0\u003e{\n                        backingStore: \u003c*store.InMemoryBackingStore | 0xc000517580\u003e{\n                            returnOnlyChangedValues: false,\n                            initializationCompleted: false,\n                            store: {\n                                \"innerError\": \u003c*odataerrors.InnerError | 0xc000741d30\u003e{\n                                    backingStore: \u003c*store.InMemoryBackingStore | 0xc0005175c0\u003e{\n                                        returnOnlyChangedValues: false,\n                                        initializationCompleted: false,\n                                        store: {...: ..., ...: ...},\n                                        subscribers: {},\n                                        changedValues: {...: ..., ...: ...},\n                                    },\n                                },\n                                \"code\": \u003c*string | 0xc000741a20\u003e\"Request_ResourceNotFound\",\n                                \"message\": \u003c*string | 0xc000741a70\u003e\"Resource '9c86f191-fe0d-40fc-8883-79c3daf62dcd' does not exist or one of its queried reference-property objects are not present.\",\n                            },\n                            subscribers: {},\n                            changedValues: {\"innerError\": false, \"code\": false, \"message\": false},\n                        },\n                    },\n                },\n                subscribers: {},\n                changedValues: {\n                    \"errorEscaped\": false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:252]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000517600\u003e: \n    add password: error status code received from the API\n    {\n        msg: \"add password: error status code received from the API\",\n        err: \u003c*odataerrors.ODataError | 0xc000b0ba40\u003e{\n            ApiError: {\n                Message: \"\",\n                ResponseStatusCode: 404,\n                ResponseHeaders: {\n                    header: {\n                        headers: {\n                            \"client-request-id\": {\n                                \"0c31e575-0500-4a44-aebc-155ee4c1eea2\": {},\n                            },\n                            \"content-type\": {\n                                \"application/json\": {},\n                            },\n                            \"x-ms-resource-unit\": {\"1\": {}},\n                            \"date\": {\n                                \"Tue, 31 Mar 2026 02:30:32 GMT\": {},\n                            },\n                            \"strict-transport-security\": {\n                                \"max-age=31536000\": {},\n                            },\n                            \"request-id\": {\n                                \"0c31e575-0500-4a44-aebc-155ee4c1eea2\": {},\n                            },\n                            \"x-ms-ags-diagnostic\": {\n                                \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"Canada Central\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"3\\\",\\\"ScaleUnit\\\":\\\"001\\\",\\\"RoleInstance\\\":\\\"YT1PEPF00003E4B\\\"}}\": {},\n                            },\n                            \"cache-control\": {\"no-cache\": {}},\n                            \"vary\": {\n                                \"Accept-Encoding\": {},\n                            },\n                        },\n                    },\n                },\n            },\n            backingStore: \u003c*store.InMemoryBackingStore | 0xc000517540\u003e{\n                returnOnlyChangedValues: false,\n                initializationCompleted: false,\n                store: {\n                    \"errorEscaped\": \u003c*odataerrors.MainError | 0xc000741cd0\u003e{\n                        backingStore: \u003c*store.InMemoryBackingStore | 0xc000517580\u003e{\n                            returnOnlyChangedValues: false,\n                            initializationCompleted: false,\n                            store: {\n                                \"innerError\": \u003c*odataerrors.InnerError | 0xc000741d30\u003e{\n                                    backingStore: \u003c*store.InMemoryBackingStore | 0xc0005175c0\u003e{\n                                        returnOnlyChangedValues: false,\n                                        initializationCompleted: false,\n                                        store: {...: ..., ...: ...},\n                                        subscribers: {},\n                                        changedValues: {...: ..., ...: ...},\n                                    },\n                                },\n                                \"code\": \u003c*string | 0xc000741a20\u003e\"Request_ResourceNotFound\",\n                                \"message\": \u003c*string | 0xc000741a70\u003e\"Resource '9c86f191-fe0d-40fc-8883-79c3daf62dcd' does not exist or one of its queried reference-property objects are not present.\",\n                            },\n                            subscribers: {},\n                            changedValues: {\"innerError\": false, \"code\": false, \"message\": false},\n                        },\n                    },\n                },\n                subscribers: {},\n                changedValues: {\n                    \"errorEscaped\": false,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038798596739960832","occurred_at":"2026-03-31T02:00:39Z","signature_id":"dcd139fb1ebeeddda77d0590aec0556e86c5a78307a2bbe487575005cdbd94b6","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}