{"meta":{"start_date":"2026-03-31","end_date":"2026-03-31","start_at":"2026-03-31T00:00:00Z","end_at":"2026-04-01T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T19:05:17Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":1,"failed_runs":1,"raw_occurrences":3,"matched_occurrences":3,"runs_affected":1},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-k686r9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":33,"trend_counts":[0,5,1,1,0,8,2],"trend_range":"2026-03-25..2026-03-31 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":17,"prior_last_seen_at":"2026-03-28T02:00:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:136]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00140c7b0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-k686r9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-k686r9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f718d0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00140c780\u003e{\n                msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-k686r9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f718d0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:136]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00140c7b0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-k686r9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-k686r9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f718d0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00140c780\u003e{\n                msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-k686r9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f718d0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-k686r9\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010a6060\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-gtwhlj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-gtwhlj, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-gtwhlj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-gtwhlj, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0016896f0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-gtwhlj\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010a6060\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-gtwhlj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-gtwhlj, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-gtwhlj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-gtwhlj, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0016896f0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-gtwhlj\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038798596630908928","occurred_at":"2026-03-31T02:00:39Z","signature_id":"405986c2270dd2c9a58a9eefe9a0d0c8d508931c1b2989d0ffe4c96875a78179","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038798596630908928","occurred_at":"2026-03-31T02:00:39Z","signature_id":"f802eb932347a79c5b8c8ef86733787164f1458619a6ece3d088789502fac787","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ee1c062c00e1e61daa37b83a9433bafe2cf1df398e225f1de7fddf080af4e874","failure_pattern":"add password: error status code received from the API","search_query":"add password: error status code received from the API","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-25..2026-03-31 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:130]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008802e0\u003e: \n    add password: error status code received from the API\n    {\n        msg: \"add password: error status code received from the API\",\n        err: \u003c*odataerrors.ODataError | 0xc000b91aa0\u003e{\n            ApiError: {\n                Message: \"\",\n                ResponseStatusCode: 404,\n                ResponseHeaders: {\n                    header: {\n                        headers: {\n                            \"strict-transport-security\": {\n                                \"max-age=31536000\": {},\n                            },\n                            \"client-request-id\": {\n                                \"609e61cb-e058-40bf-954c-a040be046529\": {},\n                            },\n                            \"content-type\": {\n                                \"application/json\": {},\n                            },\n                            \"x-ms-ags-diagnostic\": {\n                                \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"East US\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"5\\\",\\\"ScaleUnit\\\":\\\"001\\\",\\\"RoleInstance\\\":\\\"MN1PEPF00010855\\\"}}\": {},\n                            },\n                            \"x-ms-resource-unit\": {\"1\": {}},\n                            \"date\": {\n                                \"Tue, 31 Mar 2026 03:00:12 GMT\": {},\n                            },\n                            \"vary\": {\n                                \"Accept-Encoding\": {},\n                            },\n                            \"request-id\": {\n                                \"609e61cb-e058-40bf-954c-a040be046529\": {},\n                            },\n                            \"cache-control\": {\"no-cache\": {}},\n                        },\n                    },\n                },\n            },\n            backingStore: \u003c*store.InMemoryBackingStore | 0xc000880280\u003e{\n                returnOnlyChangedValues: false,\n                initializationCompleted: false,\n                store: {\n                    \"errorEscaped\": \u003c*odataerrors.MainError | 0xc0014a1650\u003e{\n                        backingStore: \u003c*store.InMemoryBackingStore | 0xc0008802a0\u003e{\n                            returnOnlyChangedValues: false,\n                            initializationCompleted: false,\n                            store: {\n                                \"code\": \u003c*string | 0xc0014a14d0\u003e\"Request_ResourceNotFound\",\n                                \"message\": \u003c*string | 0xc0014a1520\u003e\"Resource 'b5d9870c-3832-4af0-9062-5ced2786c558' does not exist or one of its queried reference-property objects are not present.\",\n                                \"innerError\": \u003c*odataerrors.InnerError | 0xc0014a16b0\u003e{\n                                    backingStore: \u003c*store.InMemoryBackingStore | 0xc0008802c0\u003e{\n                                        returnOnlyChangedValues: false,\n                                        initializationCompleted: false,\n                                        store: {...: ..., ...: ...},\n                                        subscribers: {},\n                                        changedValues: {...: ..., ...: ...},\n                                    },\n                                },\n                            },\n                            subscribers: {},\n                            changedValues: {\"code\": false, \"message\": false, \"innerError\": false},\n                        },\n                    },\n                },\n                subscribers: {},\n                changedValues: {\n                    \"errorEscaped\": false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:130]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008802e0\u003e: \n    add password: error status code received from the API\n    {\n        msg: \"add password: error status code received from the API\",\n        err: \u003c*odataerrors.ODataError | 0xc000b91aa0\u003e{\n            ApiError: {\n                Message: \"\",\n                ResponseStatusCode: 404,\n                ResponseHeaders: {\n                    header: {\n                        headers: {\n                            \"strict-transport-security\": {\n                                \"max-age=31536000\": {},\n                            },\n                            \"client-request-id\": {\n                                \"609e61cb-e058-40bf-954c-a040be046529\": {},\n                            },\n                            \"content-type\": {\n                                \"application/json\": {},\n                            },\n                            \"x-ms-ags-diagnostic\": {\n                                \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"East US\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"5\\\",\\\"ScaleUnit\\\":\\\"001\\\",\\\"RoleInstance\\\":\\\"MN1PEPF00010855\\\"}}\": {},\n                            },\n                            \"x-ms-resource-unit\": {\"1\": {}},\n                            \"date\": {\n                                \"Tue, 31 Mar 2026 03:00:12 GMT\": {},\n                            },\n                            \"vary\": {\n                                \"Accept-Encoding\": {},\n                            },\n                            \"request-id\": {\n                                \"609e61cb-e058-40bf-954c-a040be046529\": {},\n                            },\n                            \"cache-control\": {\"no-cache\": {}},\n                        },\n                    },\n                },\n            },\n            backingStore: \u003c*store.InMemoryBackingStore | 0xc000880280\u003e{\n                returnOnlyChangedValues: false,\n                initializationCompleted: false,\n                store: {\n                    \"errorEscaped\": \u003c*odataerrors.MainError | 0xc0014a1650\u003e{\n                        backingStore: \u003c*store.InMemoryBackingStore | 0xc0008802a0\u003e{\n                            returnOnlyChangedValues: false,\n                            initializationCompleted: false,\n                            store: {\n                                \"code\": \u003c*string | 0xc0014a14d0\u003e\"Request_ResourceNotFound\",\n                                \"message\": \u003c*string | 0xc0014a1520\u003e\"Resource 'b5d9870c-3832-4af0-9062-5ced2786c558' does not exist or one of its queried reference-property objects are not present.\",\n                                \"innerError\": \u003c*odataerrors.InnerError | 0xc0014a16b0\u003e{\n                                    backingStore: \u003c*store.InMemoryBackingStore | 0xc0008802c0\u003e{\n                                        returnOnlyChangedValues: false,\n                                        initializationCompleted: false,\n                                        store: {...: ..., ...: ...},\n                                        subscribers: {},\n                                        changedValues: {...: ..., ...: ...},\n                                    },\n                                },\n                            },\n                            subscribers: {},\n                            changedValues: {\"code\": false, \"message\": false, \"innerError\": false},\n                        },\n                    },\n                },\n                subscribers: {},\n                changedValues: {\n                    \"errorEscaped\": false,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038798596630908928","occurred_at":"2026-03-31T02:00:39Z","signature_id":"497600bff911ea045bfa7733f2a938420f421f08a6dde150fcd82db14842748e","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}