{"meta":{"start_date":"2026-03-13","end_date":"2026-03-13","start_at":"2026-03-13T00:00:00Z","end_at":"2026-03-14T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T20:37:38Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":10,"failed_runs":2,"raw_occurrences":3,"matched_occurrences":2,"runs_affected":1},"rows":[{"environment":"prod","failure_pattern_id":"c97096be45b81a607d04227385a0a3aae203e93e458d2b15d2856d286ce4aa28","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade using the candidate channel","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-07..2026-03-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade using the candidate channel","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: Timed out after 2400.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f195d8\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000cbc700\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                err: \u003c*errors.errorString | 0xc0008f4540\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: Timed out after 2400.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f195d8\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000cbc700\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                err: \u003c*errors.errorString | 0xc0008f4540\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                },\n            },\n        ],\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2032275495365644288","occurred_at":"2026-03-13T02:00:10Z","signature_id":"3cd35cb2d561c143dde902bf48e3273b5693b6674cda38a33f57c59d4c2c781a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3f8b20fc4049367013254c08e7b874167b87dd6570c75baa337e49f1af7f831d","failure_pattern":"failed to create route: the server is currently unable to handle the request","search_query":"failed to create route: the server is currently unable to handle the request","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-07..2026-03-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:131]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009bfcc0\u003e: \n    failed to create route: the server is currently unable to handle the request\n    {\n        msg: \"failed to create route: the server is currently unable to handle the request\",\n        err: \u003c*errors.StatusError | 0xc000f1fe00\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"the server is currently unable to handle the request\",\n                Reason: \"ServiceUnavailable\",\n                Details: {\n                    Name: \"\",\n                    Group: \"\",\n                    Kind: \"\",\n                    UID: \"\",\n                    Causes: [\n                        {\n                            Type: \"UnexpectedServerResponse\",\n                            Message: \"error trying to reach service: tls: first record does not look like a TLS handshake\",\n                            Field: \"\",\n                        },\n                    ],\n                    RetryAfterSeconds: 0,\n                },\n                Code: 503,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:131]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009bfcc0\u003e: \n    failed to create route: the server is currently unable to handle the request\n    {\n        msg: \"failed to create route: the server is currently unable to handle the request\",\n        err: \u003c*errors.StatusError | 0xc000f1fe00\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"the server is currently unable to handle the request\",\n                Reason: \"ServiceUnavailable\",\n                Details: {\n                    Name: \"\",\n                    Group: \"\",\n                    Kind: \"\",\n                    UID: \"\",\n                    Causes: [\n                        {\n                            Type: \"UnexpectedServerResponse\",\n                            Message: \"error trying to reach service: tls: first record does not look like a TLS handshake\",\n                            Field: \"\",\n                        },\n                    ],\n                    RetryAfterSeconds: 0,\n                },\n                Code: 503,\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2032275495365644288","occurred_at":"2026-03-13T02:00:10Z","signature_id":"954285fd93bc61498023104ca20dcd7f185a10d43766598b559838bb36d96ceb","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}