Alerts

alert: Build02ClusterHealth
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_build_cluster_pass_ratio{cluster="build02",period="twoDay"}[15m])
  < 0.7)
for: 10m
labels:
  channel: openshift4-next-b02
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Build cluster {{ $labels.cluster }} {{ $labels.period }} pass ratio
    is below threshold, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/build_clusters
  summary: '{{ $labels.name }}'

alert: ConsecutivePayloadRejectionsMultiArch
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_payloads_consecutively_rejected{architecture!="amd64",releaseStatus="Development"}[15m])
  >= 2)
for: 10m
labels:
  category: payload-health
  channel: alert-ocp-multi-arch-ci
  namespace: trt-monitoring
  severity: warning
annotations:
  description: The last {{ $value }} {{ $labels.release }} {{ $labels.stream }} {{
    $labels.architecture }} payloads have been rejected.
  link: https://{{ $labels.architecture }}.ocp.releases.ci.openshift.org/#{{ $labels.release
    }}.0-0.{{ $labels.stream }}
  summary: Consecutive Payload Rejections for {{ $labels.release }} {{ $labels.stream
    }} {{ $labels.architecture }}

alert: ConsoleComponentReadinessComponentRegression
expr: sum
  without(instance) (topk without(instance) (1, count by(component, view, instance)
  (min_over_time(sippy_component_readiness{component="Management Console",releaseStatus="Development"}[15m])
  < -1) >= 1))
for: 5m
labels:
  category: component-readiness
  channel: forum-ocp-console
  namespace: trt-monitoring
  severity: critical
annotations:
  description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
    }}'
  link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
    $labels.view }}
  summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
    }}

alert: ControlPlaneComponentReadinessComponentRegression
expr: sum
  without(instance) (topk without(instance) (1, count by(component, view, instance)
  (min_over_time(sippy_component_readiness{component="apiserver-auth",releaseStatus="Development"}[15m])
  < -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
  view, instance) (min_over_time(sippy_component_readiness{component="Etcd",releaseStatus="Development"}[15m])
  < -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
  view, instance) (min_over_time(sippy_component_readiness{component="Image Registry",releaseStatus="Development"}[15m])
  < -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
  view, instance) (min_over_time(sippy_component_readiness{component="kube-apiserver",releaseStatus="Development"}[15m])
  < -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
  view, instance) (min_over_time(sippy_component_readiness{component="kube-controller-manager",releaseStatus="Development"}[15m])
  < -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
  view, instance) (min_over_time(sippy_component_readiness{component="kube-scheduler",releaseStatus="Development"}[15m])
  < -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
  view, instance) (min_over_time(sippy_component_readiness{component="oauth-apiserver",releaseStatus="Development"}[15m])
  < -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
  view, instance) (min_over_time(sippy_component_readiness{component="oc",releaseStatus="Development"}[15m])
  < -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
  view, instance) (min_over_time(sippy_component_readiness{component="openshift-apiserver",releaseStatus="Development"}[15m])
  < -1) >= 1))
for: 5m
labels:
  category: component-readiness
  channel: control-plane-cr
  namespace: trt-monitoring
  severity: critical
annotations:
  description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
    }}'
  link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
    $labels.view }}
  summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
    }}

alert: DisruptionRegressionP50
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_disruption_vs_prev_ga{delta="P50",platform!="azure",releaseStatus="Development"}[15m])
  >= 1 and ignoring(delta) max_over_time(sippy_disruption_vs_prev_ga_relevance[15m])
  >= 2)
for: 5m
labels:
  category: disruption
  namespace: trt-monitoring
  severity: warning
annotations:
  description: 'P50 disruption has regressed over the past 3 days when compared to
    the 30 days prior to previous GA release for: {{ $labels.backend }} {{ $labels.platform
    }} {{ $labels.upgrade_type }} {{ $labels.network }} '
  link: https://grafana-loki.ci.openshift.org/d/ISnBj4LVk/disruption?orgId=1&var-percentile=P50&var-platform={{
    $labels.platform }}&var-backend={{ $labels.backend }}&var-upgrade_type={{
    $labels.upgrade_type }}&var-master_nodes_updated={{ $labels.master_nodes_updated
    }}&var-architectures={{ $labels.architecture }}&var-topologies={{ $labels.topology
    }}&var-networks={{ $labels.network }}&var-releases={{ $labels.release
    }}
  summary: 'P50 disruption over the past 3 days is at least 1s worse than previous
    GA release '

alert: HoursSincePayloadMultiArch
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_payloads_hours_since_last_accepted{architecture!="amd64",releaseStatus="Development"}[15m])
  >= 24)
for: 10m
labels:
  category: payload-health
  channel: alert-ocp-multi-arch-ci
  namespace: trt-monitoring
  severity: warning
annotations:
  description: It has been {{ $value | printf "%.2f" }} hours since the last
    {{ $labels.release }} {{ $labels.stream }} {{ $labels.architecture }} payload
    was accepted.
  link: https://{{ $labels.architecture }}.ocp.releases.ci.openshift.org/#{{ $labels.release
    }}.0-0.{{ $labels.stream }}
  summary: Payload Acceptance Delay for {{ $labels.release }} {{ $labels.stream }}
    {{ $labels.architecture }}

alert: TestFrameworkComponentReadinessComponentRegression
expr: sum
  without(instance) (topk without(instance) (1, count by(component, view, instance)
  (min_over_time(sippy_component_readiness{component="Test Framework",releaseStatus="Development"}[15m])
  < -1) >= 1))
for: 5m
labels:
  category: component-readiness
  namespace: trt-monitoring
  severity: critical
annotations:
  description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
    }}'
  link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
    $labels.view }}
  summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
    }}

alert: UnknownComponentReadinessComponentRegression
expr: sum
  without(instance) (topk without(instance) (1, count by(component, view, instance)
  (min_over_time(sippy_component_readiness{component="Unknown",releaseStatus="Development"}[15m])
  < -1) >= 1))
for: 5m
labels:
  category: component-readiness
  namespace: trt-monitoring
  severity: critical
annotations:
  description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
    }}'
  link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
    $labels.view }}
  summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
    }}

alert: aws-ovn-serial
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic.*e2e-aws-ovn-serial",period="twoDay",silenced="false"}[15m])
  < 0.6)
for: 1h30m
labels:
  channel: ovnk-tiger-ci
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.6, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: azure-upgrade-deploy-cnv
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-azure-upgrade-cnv",period="twoDay",release!~"4.6|4.7|4.8",silenced="false"}[15m])
  < 0.7) or avg without(instance, pod, container, namespace, endpoint, service)
  (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-azure-deploy-cnv",period="twoDay",release!~"4.6|4.7|4.8",silenced="false"}[15m])
  < 0.7)
for: 10m
labels:
  channel: cnv-operators
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: credentials-request-freeze
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~".+credentials-request-freeze",period="twoDay",silenced="false"}[15m])
  < 0.9)
for: 1h30m
labels:
  channel: forum-ocp-hive
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.9, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: e2e-aws-driver-toolkit
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~".+e2e-aws-driver-toolkit",period="current",silenced="false"}[15m])
  < 0.7)
for: 10m
labels:
  channel: dtk-ci
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.7, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: e2e-aws-sdn-cgroupsv2
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-aws-sdn-cgroupsv2",period="twoDay",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.75)
for: 10m
labels:
  channel: forum-node-jira
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.75, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: e2e-control-plane-machine-set-operator
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-cluster-control-plane-machine-set-operator-release.*periodics-e2e-aws",period="current",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.8)
  or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-cluster-control-plane-machine-set-operator-release.*periodics-e2e-aws-arm",period="current",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.8)
  or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-cluster-control-plane-machine-set-operator-release.*periodics-e2e-azure",period="current",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.8)
  or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-cluster-control-plane-machine-set-operator-release.*periodics-e2e-gcp",period="current",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.8)
for: 1h30m
labels:
  channel: forum-cloud-alerts
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.8, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: e2e-hypershift
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-hypershift-release.*periodics-e2e-aws-ovn",period="twoDay",releaseStatus=~"Full
  Support|Development",silenced="false"}[15m]) < 0.6) or avg without(instance,
  pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-hypershift-release.*periodics-e2e-aws-multi",period="twoDay",releaseStatus=~"Full
  Support|Development",silenced="false"}[15m]) < 0.6)
for: 1h30m
labels:
  channel: team-ocp-hypershift-alerts
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.6, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: e2e-metal-ipi-serial
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-metal-ipi-serial-ovn-ipv6",period="twoDay",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.5)
  or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-metal-ipi-serial-virtualmedia",period="twoDay",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.5)
  or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-metal-ipi-serial-ovn-dualstack",period="twoDay",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.5)
for: 1h30m
labels:
  channel: forum-metal-alerts
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.5, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: e2e-osd-ccs-gcp
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-osd-ccs-gcp",period="twoDay",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.75)
for: 10m
labels:
  channel: wg-hcm-continuous-ocp-release-validation
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.75, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: e2e-rosa-sts-hypershift-ovn
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-rosa-sts-hypershift-ovn",period="twoDay",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.75)
for: 10m
labels:
  channel: wg-hcm-continuous-ocp-release-validation
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.75, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: e2e-rosa-sts-ovn
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-rosa-sts-ovn",period="twoDay",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.75)
for: 10m
labels:
  channel: wg-hcm-continuous-ocp-release-validation
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.75, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: e2e-telco5g
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-telco5g",period="twoDay",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.8)
for: 1h30m
labels:
  channel: cnf-ci-reports
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.8, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: etcd-ci-alerts
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-azure-ovn-etcd-scaling",period="current",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
  or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-aws-ovn-etcd-scaling",period="current",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
  or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-vsphere-ovn-etcd-scaling",period="current",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
  or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-gcp-ovn-etcd-scaling",period="current",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
for: 1h30m
labels:
  channel: etcd-ci-alerts
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.7, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: hcm-cicd-alerts
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*osd-aws",period="current",releaseStatus=~"(Maintenance
  Support|Full Support|Development)",silenced="false"}[2d]) < 0.7 and
  count_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*osd-aws",period="current",releaseStatus=~"(Maintenance
  Support|Full Support|Development)",silenced="false"}[2d]) > 0) or
  avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*osd-gcp",period="current",releaseStatus=~"(Maintenance
  Support|Full Support|Development)",silenced="false"}[2d]) < 0.7 and
  count_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*osd-gcp",period="current",releaseStatus=~"(Maintenance
  Support|Full Support|Development)",silenced="false"}[2d]) > 0) or
  avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*rosa-classic-sts",period="current",releaseStatus=~"(Maintenance
  Support|Full Support|Development)",silenced="false"}[2d]) < 0.7 and
  count_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*rosa-classic-sts",period="current",releaseStatus=~"(Maintenance
  Support|Full Support|Development)",silenced="false"}[2d]) > 0) or
  avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*rosa-hcp",period="current",releaseStatus=~"(Maintenance
  Support|Full Support|Development)",silenced="false"}[2d]) < 0.7 and
  count_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*rosa-hcp",period="current",releaseStatus=~"(Maintenance
  Support|Full Support|Development)",silenced="false"}[2d]) > 0)
for: 1d
labels:
  channel: hcm-cicd-alerts
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.7, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: recert-ci
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly-.*-metal-ovn-single-node-recert-cluster-rename",period="twoDay",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
for: 1h30m
labels:
  channel: recert-ci-alerts
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.7, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: single-node-job-pass-ratio
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~".*-e2e-aws-ovn-single-node",period="twoDay",releaseStatus=~"Full
  Support|Development",silenced="false"}[15m]) < 0.9)
for: 1h30m
labels:
  channel: ci-single-node
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.9, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: Job pass ratio for {{ $labels.name }}

alert: telcov10n-ci
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-kni-eco-ci-cd-main-nightly-.*-telcov10n-metal-single-node-spoke",period="twoDay",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
for: 1h30m
labels:
  channel: telco-verification-alerts
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.7, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: DisruptionRegressionP95
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_disruption_vs_prev_ga{delta="P95",releaseStatus="Development"}[15m])
  >= 5 and ignoring(delta) max_over_time(sippy_disruption_vs_prev_ga_relevance[15m])
  >= 2)
for: 3d
labels:
  category: disruption
  namespace: trt-monitoring
  severity: warning
annotations:
  description: 'P95 disruption has regressed over the past several days when compared
    to the 30 days prior to previous GA release for: {{ $labels.backend }} {{ $labels.platform
    }} {{ $labels.upgrade_type }} {{ $labels.network }} '
  link: https://grafana-loki.ci.openshift.org/d/ISnBj4LVk/disruption?orgId=1&var-percentile=P95&var-platform={{
    $labels.platform }}&var-backend={{ $labels.backend }}&var-upgrade_type={{
    $labels.upgrade_type }}&var-master_nodes_updated={{ $labels.master_nodes_updated
    }}&var-architectures={{ $labels.architecture }}&var-topologies={{ $labels.topology
    }}&var-networks={{ $labels.network }}&var-releases={{ $labels.release
    }}
  summary: 'P95 disruption over the past 3 days is at least 5s worse than previous
    GA release '

alert: CVOComponentReadinessComponentRegression
expr: sum
  without(instance) (topk without(instance) (1, count by(component, view, instance)
  (min_over_time(sippy_component_readiness{component="Cluster Version Operator",releaseStatus="Development"}[15m])
  < -1) >= 1))
for: 5m
labels:
  category: component-readiness
  channel: forum-ocp-updates
  namespace: trt-monitoring
  severity: critical
annotations:
  description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
    }}'
  link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
    $labels.view }}
  summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
    }}

alert: DNSComponentReadinessComponentRegression
expr: sum
  without(instance) (topk without(instance) (1, count by(component, view, instance)
  (min_over_time(sippy_component_readiness{component="Networking / DNS",releaseStatus="Development"}[15m])
  < -1) >= 1))
for: 5m
labels:
  category: component-readiness
  channel: forum-ocp-network-edge
  namespace: trt-monitoring
  severity: critical
annotations:
  description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
    }}'
  link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
    $labels.view }}
  summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
    }}

alert: DisruptionRegressionP50Azure
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_disruption_vs_prev_ga{delta="P50",platform="azure",releaseStatus="Development"}[15m])
  >= 2 and ignoring(delta) max_over_time(sippy_disruption_vs_prev_ga_relevance[15m])
  >= 2)
for: 5m
labels:
  category: disruption
  namespace: trt-monitoring
  severity: warning
annotations:
  description: 'P50 disruption has regressed over the past 3 days when compared to
    the 30 days prior to previous GA release for: {{ $labels.backend }} {{ $labels.platform
    }} {{ $labels.upgrade_type }} {{ $labels.network }} '
  link: https://grafana-loki.ci.openshift.org/d/ISnBj4LVk/disruption?orgId=1&var-percentile=P50&var-platform={{
    $labels.platform }}&var-backend={{ $labels.backend }}&var-upgrade_type={{
    $labels.upgrade_type }}&var-master_nodes_updated={{ $labels.master_nodes_updated
    }}&var-architectures={{ $labels.architecture }}&var-topologies={{ $labels.topology
    }}&var-networks={{ $labels.network }}&var-releases={{ $labels.release
    }}
  summary: 'P50 disruption over the past 3 days is at least 1s worse than previous
    GA release '

alert: DisruptionRegressionP75
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_disruption_vs_prev_ga{delta="P75",releaseStatus="Development"}[15m])
  >= 3 and ignoring(delta) max_over_time(sippy_disruption_vs_prev_ga_relevance[15m])
  >= 2)
for: 2d
labels:
  category: disruption
  namespace: trt-monitoring
  severity: warning
annotations:
  description: 'P75 disruption has regressed over the past several days when compared
    to the 30 days prior to previous GA release for: {{ $labels.backend }} {{ $labels.platform
    }} {{ $labels.upgrade_type }} {{ $labels.network }} '
  link: https://grafana-loki.ci.openshift.org/d/ISnBj4LVk/disruption?orgId=1&var-percentile=P75&var-platform={{
    $labels.platform }}&var-backend={{ $labels.backend }}&var-upgrade_type={{
    $labels.upgrade_type }}&var-master_nodes_updated={{ $labels.master_nodes_updated
    }}&var-architectures={{ $labels.architecture }}&var-topologies={{ $labels.topology
    }}&var-networks={{ $labels.network }}&var-releases={{ $labels.release
    }}
  summary: 'P75 disruption over the past 3 days is at least 3s worse than previous
    GA release '

alert: GitOpsZTPComponentReadinessComponentRegression
expr: sum
  without(instance) (topk without(instance) (1, count by(component, view, instance)
  (min_over_time(sippy_component_readiness{component="GitOps ZTP",releaseStatus="Development"}[15m])
  < -1) >= 1))
for: 5m
labels:
  category: component-readiness
  channel: telco-verification-alerts
  namespace: trt-monitoring
  severity: critical
annotations:
  description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
    }}'
  link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
    $labels.view }}
  summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
    }}

alert: PrometheusTargetMissing
expr: up{job="sippy"}
  < 1
for: 20m
labels:
  category: sippy-health
  namespace: trt-monitoring
  severity: critical
annotations:
  description: |-
    Sippy target has disappeared.
      VALUE = {{ $value }}
      LABELS = {{ $labels }}
  summary: Prometheus target missing (instance {{ $labels.instance }})

alert: RouterComponentReadinessComponentRegression
expr: sum
  without(instance) (topk without(instance) (1, count by(component, view, instance)
  (min_over_time(sippy_component_readiness{component="Networking / router",releaseStatus="Development"}[15m])
  < -1) >= 1))
for: 5m
labels:
  category: component-readiness
  channel: forum-ocp-network-edge
  namespace: trt-monitoring
  severity: critical
annotations:
  description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
    }}'
  link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
    $labels.view }}
  summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
    }}

alert: SippyNotUpdating
expr: max_over_time(sippy_hours_since_last_update[15m])
  >= 2
for: 10m
labels:
  category: sippy-health
  namespace: trt-monitoring
  severity: critical
annotations:
  description: It has been {{ $value | printf "%.2f" }} hours since the last
    update.
  summary: Sippy is not updating!

alert: StorageComponentReadinessComponentRegression
expr: sum
  without(instance) (topk without(instance) (1, count by(component, view, instance)
  (min_over_time(sippy_component_readiness{component="Storage",releaseStatus="Development"}[15m])
  < -1) >= 1))
for: 5m
labels:
  category: component-readiness
  channel: team-ocp-storage
  namespace: trt-monitoring
  severity: critical
annotations:
  description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
    }}'
  link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
    $labels.view }}
  summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
    }}

alert: WorkloadsComponentReadinessComponentRegression
expr: sum
  without(instance) (topk without(instance) (1, count by(component, view, instance)
  (min_over_time(sippy_component_readiness{component="oc",releaseStatus="Development"}[15m])
  < -1) >= 1))
for: 5m
labels:
  category: component-readiness
  channel: team-workloads
  namespace: trt-monitoring
  severity: critical
annotations:
  description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
    }}'
  link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
    $labels.view }}
  summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
    }}

alert: e2e-agent
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic.*e2e-agent-compact-ipv4",period="twoDay",releaseStatus=~"Full
  Support|Development",silenced="false"}[15m]) < 0.7) or avg without(instance,
  pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic.*e2e-agent-ha-dualstack",period="twoDay",releaseStatus=~"Full
  Support|Development",silenced="false"}[15m]) < 0.7) or avg without(instance,
  pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic.*e2e-agent-sno-ipv6",period="twoDay",releaseStatus=~"Maintenance
  Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
for: 1h30m
labels:
  channel: installer-alerts
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.7, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: e2e-aws-workers-rhel8
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic.*e2e-aws-workers-rhel8",period="current",releaseStatus=~"Full
  Support|Development",silenced="false"}[15m]) < 0.7)
for: 1h30m
labels:
  channel: installer-alerts
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.7, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: qe-perfscale
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-qe-ocp-qe-perfscale-ci-main-aws-.*-nightly-x86-payload-control-plane-6nodes",period="twoDay",releaseStatus=~"Development",silenced="false"}[15m])
  < 0.9)
for: 1h30m
labels:
  channel: ocp-qe-scale-ci-results-alerts
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 0.9, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'

alert: single-node-install-success-rate
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_install_success_last{period="twoDay",releaseStatus=~"Development",variant="Topology:single"}[15m])
  < 90) or avg without(instance, pod, container, namespace, endpoint, service)
  (max_over_time(sippy_install_success_last{period="current",releaseStatus=~"Maintenance
  Support|Full Support",variant="Topology:single"}[15m]) < 90)
for: 10m
labels:
  channel: ci-single-node
  namespace: trt-monitoring
  severity: warning
annotations:
  description: '{{ $labels.release }} for period {{ $labels.period }} pass ratio is
    below threshold, expecting 90%, got {{ $value }}%.'
  link: 'https://sippy.dptools.openshift.org/sippy-ng/tests/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"install
    should succeed: overall"},{"columnField":"variants","operatorValue":"contains","value":"upgrade-minor","not":true},{"columnField":"variants","operatorValue":"contains","value":"aggregated","not":true},{"columnField":"variants","operatorValue":"contains","value":"never-stable","not":true},{"columnField":"variants","operatorValue":"contains","value":"Topology:single"}],"linkOperator":"and"}&test=install%20should%20succeed%3A%20overall'
  summary: Single Node install success rate for {{ $labels.release }} is low

alert: vsphere-sdn-ipi-install-analysis
expr: avg
  without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~".+install-analysis-vsphere-sdn-ipi",period="current",silenced="false"}[15m])
  < 1)
for: 1h30m
labels:
  channel: vmc-vsphere-alerts
  namespace: trt-monitoring
  severity: warning
annotations:
  description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
    expecting 1, got {{ $value | printf "%.2f" }}.
  link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
    $labels.name }}"}]}
  summary: '{{ $labels.name }}'