| Rule |
State |
Error |
Last Evaluation |
Evaluation Time |
| alert: PrometheusTargetMissing
expr: up{job="sippy"}
< 1
for: 20m
labels:
category: sippy-health
namespace: trt-monitoring
severity: critical
annotations:
description: |-
Sippy target has disappeared.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Prometheus target missing (instance {{ $labels.instance }})
|
ok
|
|
26.576s ago
|
331.9us |
| alert: SippyNotUpdating
expr: max_over_time(sippy_hours_since_last_update[15m])
>= 2
for: 10m
labels:
category: sippy-health
namespace: trt-monitoring
severity: critical
annotations:
description: It has been {{ $value | printf "%.2f" }} hours since the last
update.
summary: Sippy is not updating!
|
ok
|
|
26.576s ago
|
203.2us |
| alert: ConsoleComponentReadinessComponentRegression
expr: sum
without(instance) (topk without(instance) (1, count by(component, view, instance)
(min_over_time(sippy_component_readiness{component="Management Console",releaseStatus="Development"}[15m])
< -1) >= 1))
for: 5m
labels:
category: component-readiness
channel: forum-ocp-console
namespace: trt-monitoring
severity: critical
annotations:
description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
}}'
link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
$labels.view }}
summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
}}
|
ok
|
|
26.576s ago
|
628.8us |
| alert: ControlPlaneComponentReadinessComponentRegression
expr: sum
without(instance) (topk without(instance) (1, count by(component, view, instance)
(min_over_time(sippy_component_readiness{component="apiserver-auth",releaseStatus="Development"}[15m])
< -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
view, instance) (min_over_time(sippy_component_readiness{component="Etcd",releaseStatus="Development"}[15m])
< -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
view, instance) (min_over_time(sippy_component_readiness{component="Image Registry",releaseStatus="Development"}[15m])
< -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
view, instance) (min_over_time(sippy_component_readiness{component="kube-apiserver",releaseStatus="Development"}[15m])
< -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
view, instance) (min_over_time(sippy_component_readiness{component="kube-controller-manager",releaseStatus="Development"}[15m])
< -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
view, instance) (min_over_time(sippy_component_readiness{component="kube-scheduler",releaseStatus="Development"}[15m])
< -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
view, instance) (min_over_time(sippy_component_readiness{component="oauth-apiserver",releaseStatus="Development"}[15m])
< -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
view, instance) (min_over_time(sippy_component_readiness{component="oc",releaseStatus="Development"}[15m])
< -1) >= 1)) or sum without(instance) (topk without(instance) (1, count by(component,
view, instance) (min_over_time(sippy_component_readiness{component="openshift-apiserver",releaseStatus="Development"}[15m])
< -1) >= 1))
for: 5m
labels:
category: component-readiness
channel: control-plane-cr
namespace: trt-monitoring
severity: critical
annotations:
description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
}}'
link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
$labels.view }}
summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
}}
|
ok
|
|
26.576s ago
|
3.16ms |
| alert: WorkloadsComponentReadinessComponentRegression
expr: sum
without(instance) (topk without(instance) (1, count by(component, view, instance)
(min_over_time(sippy_component_readiness{component="oc",releaseStatus="Development"}[15m])
< -1) >= 1))
for: 5m
labels:
category: component-readiness
channel: team-workloads
namespace: trt-monitoring
severity: critical
annotations:
description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
}}'
link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
$labels.view }}
summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
}}
|
ok
|
|
26.573s ago
|
340.2us |
| alert: StorageComponentReadinessComponentRegression
expr: sum
without(instance) (topk without(instance) (1, count by(component, view, instance)
(min_over_time(sippy_component_readiness{component="Storage",releaseStatus="Development"}[15m])
< -1) >= 1))
for: 5m
labels:
category: component-readiness
channel: team-ocp-storage
namespace: trt-monitoring
severity: critical
annotations:
description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
}}'
link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
$labels.view }}
summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
}}
|
ok
|
|
26.572s ago
|
244.8us |
| alert: CVOComponentReadinessComponentRegression
expr: sum
without(instance) (topk without(instance) (1, count by(component, view, instance)
(min_over_time(sippy_component_readiness{component="Cluster Version Operator",releaseStatus="Development"}[15m])
< -1) >= 1))
for: 5m
labels:
category: component-readiness
channel: forum-ocp-updates
namespace: trt-monitoring
severity: critical
annotations:
description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
}}'
link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
$labels.view }}
summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
}}
|
ok
|
|
26.572s ago
|
215.5us |
| alert: TestFrameworkComponentReadinessComponentRegression
expr: sum
without(instance) (topk without(instance) (1, count by(component, view, instance)
(min_over_time(sippy_component_readiness{component="Test Framework",releaseStatus="Development"}[15m])
< -1) >= 1))
for: 5m
labels:
category: component-readiness
namespace: trt-monitoring
severity: critical
annotations:
description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
}}'
link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
$labels.view }}
summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
}}
|
ok
|
|
26.572s ago
|
442.4us |
| alert: UnknownComponentReadinessComponentRegression
expr: sum
without(instance) (topk without(instance) (1, count by(component, view, instance)
(min_over_time(sippy_component_readiness{component="Unknown",releaseStatus="Development"}[15m])
< -1) >= 1))
for: 5m
labels:
category: component-readiness
namespace: trt-monitoring
severity: critical
annotations:
description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
}}'
link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
$labels.view }}
summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
}}
|
ok
|
|
26.572s ago
|
488.8us |
| alert: DNSComponentReadinessComponentRegression
expr: sum
without(instance) (topk without(instance) (1, count by(component, view, instance)
(min_over_time(sippy_component_readiness{component="Networking / DNS",releaseStatus="Development"}[15m])
< -1) >= 1))
for: 5m
labels:
category: component-readiness
channel: forum-ocp-network-edge
namespace: trt-monitoring
severity: critical
annotations:
description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
}}'
link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
$labels.view }}
summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
}}
|
ok
|
|
26.571s ago
|
228.3us |
| alert: RouterComponentReadinessComponentRegression
expr: sum
without(instance) (topk without(instance) (1, count by(component, view, instance)
(min_over_time(sippy_component_readiness{component="Networking / router",releaseStatus="Development"}[15m])
< -1) >= 1))
for: 5m
labels:
category: component-readiness
channel: forum-ocp-network-edge
namespace: trt-monitoring
severity: critical
annotations:
description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
}}'
link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
$labels.view }}
summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
}}
|
ok
|
|
26.571s ago
|
203.9us |
| alert: GitOpsZTPComponentReadinessComponentRegression
expr: sum
without(instance) (topk without(instance) (1, count by(component, view, instance)
(min_over_time(sippy_component_readiness{component="GitOps ZTP",releaseStatus="Development"}[15m])
< -1) >= 1))
for: 5m
labels:
category: component-readiness
channel: telco-verification-alerts
namespace: trt-monitoring
severity: critical
annotations:
description: '{{ $labels.component }} is reporting a regression in {{ $labels.view
}}'
link: https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main?view={{
$labels.view }}
summary: Component readiness alert for {{ $labels.component }} in {{ $labels.view
}}
|
ok
|
|
26.571s ago
|
110.3us |
| alert: DisruptionRegressionP50
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_disruption_vs_prev_ga{delta="P50",platform!="azure",releaseStatus="Development"}[15m])
>= 1 and ignoring(delta) max_over_time(sippy_disruption_vs_prev_ga_relevance[15m])
>= 2)
for: 5m
labels:
category: disruption
namespace: trt-monitoring
severity: warning
annotations:
description: 'P50 disruption has regressed over the past 3 days when compared to
the 30 days prior to previous GA release for: {{ $labels.backend }} {{ $labels.platform
}} {{ $labels.upgrade_type }} {{ $labels.network }} '
link: https://grafana-loki.ci.openshift.org/d/ISnBj4LVk/disruption?orgId=1&var-percentile=P50&var-platform={{
$labels.platform }}&var-backend={{ $labels.backend }}&var-upgrade_type={{
$labels.upgrade_type }}&var-master_nodes_updated={{ $labels.master_nodes_updated
}}&var-architectures={{ $labels.architecture }}&var-topologies={{ $labels.topology
}}&var-networks={{ $labels.network }}&var-releases={{ $labels.release
}}
summary: 'P50 disruption over the past 3 days is at least 1s worse than previous
GA release '
|
ok
|
|
26.571s ago
|
147ms |
| alert: DisruptionRegressionP50Azure
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_disruption_vs_prev_ga{delta="P50",platform="azure",releaseStatus="Development"}[15m])
>= 2 and ignoring(delta) max_over_time(sippy_disruption_vs_prev_ga_relevance[15m])
>= 2)
for: 5m
labels:
category: disruption
namespace: trt-monitoring
severity: warning
annotations:
description: 'P50 disruption has regressed over the past 3 days when compared to
the 30 days prior to previous GA release for: {{ $labels.backend }} {{ $labels.platform
}} {{ $labels.upgrade_type }} {{ $labels.network }} '
link: https://grafana-loki.ci.openshift.org/d/ISnBj4LVk/disruption?orgId=1&var-percentile=P50&var-platform={{
$labels.platform }}&var-backend={{ $labels.backend }}&var-upgrade_type={{
$labels.upgrade_type }}&var-master_nodes_updated={{ $labels.master_nodes_updated
}}&var-architectures={{ $labels.architecture }}&var-topologies={{ $labels.topology
}}&var-networks={{ $labels.network }}&var-releases={{ $labels.release
}}
summary: 'P50 disruption over the past 3 days is at least 1s worse than previous
GA release '
|
ok
|
|
26.424s ago
|
113.4ms |
| alert: DisruptionRegressionP75
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_disruption_vs_prev_ga{delta="P75",releaseStatus="Development"}[15m])
>= 3 and ignoring(delta) max_over_time(sippy_disruption_vs_prev_ga_relevance[15m])
>= 2)
for: 2d
labels:
category: disruption
namespace: trt-monitoring
severity: warning
annotations:
description: 'P75 disruption has regressed over the past several days when compared
to the 30 days prior to previous GA release for: {{ $labels.backend }} {{ $labels.platform
}} {{ $labels.upgrade_type }} {{ $labels.network }} '
link: https://grafana-loki.ci.openshift.org/d/ISnBj4LVk/disruption?orgId=1&var-percentile=P75&var-platform={{
$labels.platform }}&var-backend={{ $labels.backend }}&var-upgrade_type={{
$labels.upgrade_type }}&var-master_nodes_updated={{ $labels.master_nodes_updated
}}&var-architectures={{ $labels.architecture }}&var-topologies={{ $labels.topology
}}&var-networks={{ $labels.network }}&var-releases={{ $labels.release
}}
summary: 'P75 disruption over the past 3 days is at least 3s worse than previous
GA release '
|
ok
|
|
26.311s ago
|
138.6ms |
| alert: DisruptionRegressionP95
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_disruption_vs_prev_ga{delta="P95",releaseStatus="Development"}[15m])
>= 5 and ignoring(delta) max_over_time(sippy_disruption_vs_prev_ga_relevance[15m])
>= 2)
for: 3d
labels:
category: disruption
namespace: trt-monitoring
severity: warning
annotations:
description: 'P95 disruption has regressed over the past several days when compared
to the 30 days prior to previous GA release for: {{ $labels.backend }} {{ $labels.platform
}} {{ $labels.upgrade_type }} {{ $labels.network }} '
link: https://grafana-loki.ci.openshift.org/d/ISnBj4LVk/disruption?orgId=1&var-percentile=P95&var-platform={{
$labels.platform }}&var-backend={{ $labels.backend }}&var-upgrade_type={{
$labels.upgrade_type }}&var-master_nodes_updated={{ $labels.master_nodes_updated
}}&var-architectures={{ $labels.architecture }}&var-topologies={{ $labels.topology
}}&var-networks={{ $labels.network }}&var-releases={{ $labels.release
}}
summary: 'P95 disruption over the past 3 days is at least 5s worse than previous
GA release '
|
ok
|
|
26.172s ago
|
142.5ms |
| alert: Build02ClusterHealth
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_build_cluster_pass_ratio{cluster="build02",period="twoDay"}[15m])
< 0.7)
for: 10m
labels:
channel: openshift4-next-b02
namespace: trt-monitoring
severity: warning
annotations:
description: Build cluster {{ $labels.cluster }} {{ $labels.period }} pass ratio
is below threshold, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/build_clusters
summary: '{{ $labels.name }}'
|
ok
|
|
26.03s ago
|
489.9us |
| alert: HoursSincePayloadMultiArch
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_payloads_hours_since_last_accepted{architecture!="amd64",releaseStatus="Development"}[15m])
>= 24)
for: 10m
labels:
category: payload-health
channel: alert-ocp-multi-arch-ci
namespace: trt-monitoring
severity: warning
annotations:
description: It has been {{ $value | printf "%.2f" }} hours since the last
{{ $labels.release }} {{ $labels.stream }} {{ $labels.architecture }} payload
was accepted.
link: https://{{ $labels.architecture }}.ocp.releases.ci.openshift.org/#{{ $labels.release
}}.0-0.{{ $labels.stream }}
summary: Payload Acceptance Delay for {{ $labels.release }} {{ $labels.stream }}
{{ $labels.architecture }}
|
ok
|
|
26.029s ago
|
2.092ms |
| alert: ConsecutivePayloadRejectionsMultiArch
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_payloads_consecutively_rejected{architecture!="amd64",releaseStatus="Development"}[15m])
>= 2)
for: 10m
labels:
category: payload-health
channel: alert-ocp-multi-arch-ci
namespace: trt-monitoring
severity: warning
annotations:
description: The last {{ $value }} {{ $labels.release }} {{ $labels.stream }} {{
$labels.architecture }} payloads have been rejected.
link: https://{{ $labels.architecture }}.ocp.releases.ci.openshift.org/#{{ $labels.release
}}.0-0.{{ $labels.stream }}
summary: Consecutive Payload Rejections for {{ $labels.release }} {{ $labels.stream
}} {{ $labels.architecture }}
|
ok
|
|
26.027s ago
|
764.7us |
| alert: credentials-request-freeze
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~".+credentials-request-freeze",period="twoDay",silenced="false"}[15m])
< 0.9)
for: 1h30m
labels:
channel: forum-ocp-hive
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.9, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
26.027s ago
|
854.9us |
| alert: e2e-aws-driver-toolkit
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~".+e2e-aws-driver-toolkit",period="current",silenced="false"}[15m])
< 0.7)
for: 10m
labels:
channel: dtk-ci
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.7, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
26.026s ago
|
1.851ms |
| alert: e2e-aws-sdn-cgroupsv2
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-aws-sdn-cgroupsv2",period="twoDay",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.75)
for: 10m
labels:
channel: forum-node-jira
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.75, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
26.024s ago
|
1.507ms |
| alert: e2e-metal-ipi-serial
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-metal-ipi-serial-ovn-ipv6",period="twoDay",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.5)
or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-metal-ipi-serial-virtualmedia",period="twoDay",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.5)
or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-metal-ipi-serial-ovn-dualstack",period="twoDay",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.5)
for: 1h30m
labels:
channel: forum-metal-alerts
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.5, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
26.023s ago
|
2.246ms |
| alert: aws-ovn-serial
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic.*e2e-aws-ovn-serial",period="twoDay",silenced="false"}[15m])
< 0.6)
for: 1h30m
labels:
channel: ovnk-tiger-ci
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.6, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
26.021s ago
|
4.085ms |
| alert: azure-upgrade-deploy-cnv
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-azure-upgrade-cnv",period="twoDay",release!~"4.6|4.7|4.8",silenced="false"}[15m])
< 0.7) or avg without(instance, pod, container, namespace, endpoint, service)
(max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-azure-deploy-cnv",period="twoDay",release!~"4.6|4.7|4.8",silenced="false"}[15m])
< 0.7)
for: 10m
labels:
channel: cnv-operators
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
26.017s ago
|
1.829ms |
| alert: vsphere-sdn-ipi-install-analysis
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~".+install-analysis-vsphere-sdn-ipi",period="current",silenced="false"}[15m])
< 1)
for: 1h30m
labels:
channel: vmc-vsphere-alerts
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 1, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
26.015s ago
|
295.3us |
| alert: e2e-hypershift
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-hypershift-release.*periodics-e2e-aws-ovn",period="twoDay",releaseStatus=~"Full
Support|Development",silenced="false"}[15m]) < 0.6) or avg without(instance,
pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-hypershift-release.*periodics-e2e-aws-multi",period="twoDay",releaseStatus=~"Full
Support|Development",silenced="false"}[15m]) < 0.6)
for: 1h30m
labels:
channel: team-ocp-hypershift-alerts
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.6, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
26.015s ago
|
1.061ms |
| alert: e2e-telco5g
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-telco5g",period="twoDay",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.8)
for: 1h30m
labels:
channel: cnf-ci-reports
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.8, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
26.014s ago
|
1.356ms |
| alert: etcd-ci-alerts
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-azure-ovn-etcd-scaling",period="current",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-aws-ovn-etcd-scaling",period="current",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-vsphere-ovn-etcd-scaling",period="current",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-gcp-ovn-etcd-scaling",period="current",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
for: 1h30m
labels:
channel: etcd-ci-alerts
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.7, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
26.013s ago
|
6.552ms |
| alert: hcm-cicd-alerts
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*osd-aws",period="current",releaseStatus=~"(Maintenance
Support|Full Support|Development)",silenced="false"}[2d]) < 0.7 and
count_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*osd-aws",period="current",releaseStatus=~"(Maintenance
Support|Full Support|Development)",silenced="false"}[2d]) > 0) or
avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*osd-gcp",period="current",releaseStatus=~"(Maintenance
Support|Full Support|Development)",silenced="false"}[2d]) < 0.7 and
count_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*osd-gcp",period="current",releaseStatus=~"(Maintenance
Support|Full Support|Development)",silenced="false"}[2d]) > 0) or
avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*rosa-classic-sts",period="current",releaseStatus=~"(Maintenance
Support|Full Support|Development)",silenced="false"}[2d]) < 0.7 and
count_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*rosa-classic-sts",period="current",releaseStatus=~"(Maintenance
Support|Full Support|Development)",silenced="false"}[2d]) > 0) or
avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*rosa-hcp",period="current",releaseStatus=~"(Maintenance
Support|Full Support|Development)",silenced="false"}[2d]) < 0.7 and
count_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-osde2e-main-nightly.*rosa-hcp",period="current",releaseStatus=~"(Maintenance
Support|Full Support|Development)",silenced="false"}[2d]) > 0)
for: 1d
labels:
channel: hcm-cicd-alerts
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.7, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
26.006s ago
|
51.01ms |
| alert: e2e-control-plane-machine-set-operator
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-cluster-control-plane-machine-set-operator-release.*periodics-e2e-aws",period="current",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.8)
or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-cluster-control-plane-machine-set-operator-release.*periodics-e2e-aws-arm",period="current",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.8)
or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-cluster-control-plane-machine-set-operator-release.*periodics-e2e-azure",period="current",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.8)
or avg without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-cluster-control-plane-machine-set-operator-release.*periodics-e2e-gcp",period="current",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.8)
for: 1h30m
labels:
channel: forum-cloud-alerts
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.8, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
25.955s ago
|
5.028ms |
| alert: e2e-aws-workers-rhel8
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic.*e2e-aws-workers-rhel8",period="current",releaseStatus=~"Full
Support|Development",silenced="false"}[15m]) < 0.7)
for: 1h30m
labels:
channel: installer-alerts
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.7, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
25.951s ago
|
392.8us |
| alert: e2e-agent
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic.*e2e-agent-compact-ipv4",period="twoDay",releaseStatus=~"Full
Support|Development",silenced="false"}[15m]) < 0.7) or avg without(instance,
pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic.*e2e-agent-ha-dualstack",period="twoDay",releaseStatus=~"Full
Support|Development",silenced="false"}[15m]) < 0.7) or avg without(instance,
pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic.*e2e-agent-sno-ipv6",period="twoDay",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
for: 1h30m
labels:
channel: installer-alerts
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.7, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
25.95s ago
|
753.5us |
| alert: qe-perfscale
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-qe-ocp-qe-perfscale-ci-main-aws-.*-nightly-x86-payload-control-plane-6nodes",period="twoDay",releaseStatus=~"Development",silenced="false"}[15m])
< 0.9)
for: 1h30m
labels:
channel: ocp-qe-scale-ci-results-alerts
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.9, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
25.95s ago
|
302.1us |
| alert: single-node-job-pass-ratio
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~".*-e2e-aws-ovn-single-node",period="twoDay",releaseStatus=~"Full
Support|Development",silenced="false"}[15m]) < 0.9)
for: 1h30m
labels:
channel: ci-single-node
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.9, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: Job pass ratio for {{ $labels.name }}
|
ok
|
|
25.949s ago
|
1.18ms |
| alert: single-node-install-success-rate
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_install_success_last{period="twoDay",releaseStatus=~"Development",variant="Topology:single"}[15m])
< 90) or avg without(instance, pod, container, namespace, endpoint, service)
(max_over_time(sippy_install_success_last{period="current",releaseStatus=~"Maintenance
Support|Full Support",variant="Topology:single"}[15m]) < 90)
for: 10m
labels:
channel: ci-single-node
namespace: trt-monitoring
severity: warning
annotations:
description: '{{ $labels.release }} for period {{ $labels.period }} pass ratio is
below threshold, expecting 90%, got {{ $value }}%.'
link: 'https://sippy.dptools.openshift.org/sippy-ng/tests/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"install
should succeed: overall"},{"columnField":"variants","operatorValue":"contains","value":"upgrade-minor","not":true},{"columnField":"variants","operatorValue":"contains","value":"aggregated","not":true},{"columnField":"variants","operatorValue":"contains","value":"never-stable","not":true},{"columnField":"variants","operatorValue":"contains","value":"Topology:single"}],"linkOperator":"and"}&test=install%20should%20succeed%3A%20overall'
summary: Single Node install success rate for {{ $labels.release }} is low
|
ok
|
|
25.948s ago
|
183.8us |
| alert: recert-ci
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly-.*-metal-ovn-single-node-recert-cluster-rename",period="twoDay",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
for: 1h30m
labels:
channel: recert-ci-alerts
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.7, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
25.948s ago
|
792.2us |
| alert: e2e-rosa-sts-ovn
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-rosa-sts-ovn",period="twoDay",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.75)
for: 10m
labels:
channel: wg-hcm-continuous-ocp-release-validation
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.75, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
25.948s ago
|
986.3us |
| alert: e2e-rosa-sts-hypershift-ovn
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-rosa-sts-hypershift-ovn",period="twoDay",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.75)
for: 10m
labels:
channel: wg-hcm-continuous-ocp-release-validation
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.75, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
25.947s ago
|
1.03ms |
| alert: e2e-osd-ccs-gcp
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-release-master-nightly.*e2e-osd-ccs-gcp",period="twoDay",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.75)
for: 10m
labels:
channel: wg-hcm-continuous-ocp-release-validation
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.75, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
25.946s ago
|
1.012ms |
| alert: telcov10n-ci
expr: avg
without(instance, pod, container, namespace, endpoint, service) (max_over_time(sippy_job_pass_ratio{name=~"periodic-ci-openshift-kni-eco-ci-cd-main-nightly-.*-telcov10n-metal-single-node-spoke",period="twoDay",releaseStatus=~"Maintenance
Support|Full Support|Development",silenced="false"}[15m]) < 0.7)
for: 1h30m
labels:
channel: telco-verification-alerts
namespace: trt-monitoring
severity: warning
annotations:
description: Job {{ $labels.name }} {{ $labels.period }} pass ratio is below threshold,
expecting 0.7, got {{ $value | printf "%.2f" }}.
link: https://sippy.dptools.openshift.org/sippy-ng/jobs/{{ $labels.release }}/analysis?filters={"items":[{"columnField":"name","operatorValue":"equals","value":"{{
$labels.name }}"}]}
summary: '{{ $labels.name }}'
|
ok
|
|
25.945s ago
|
973.5us |