apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  creationTimestamp: null
  labels:
    prometheus: stf-default
    role: alert-rules
  name: prometheus-alarm-rules
  namespace: service-telemetry
spec:
  groups:
    - name: ./openstack.rules
      rules:
        - expr: 'rate(collectd_disk_disk_io_time_io_time_total[5m])'
          record: 'job:iotime:rate_5m'
        - expr: 'stddev_over_time(job:iotime:rate_5m[1h])'
          record: 'job:iotime:rate_5m:stddev_over_time_1h'
        - expr: 'avg_over_time(job:iotime:rate_5m[1h])'
          record: 'job:iotime:rate_5m:avg_over_time_1h'
        - alert: iotime warn
          expr: >-
            (abs(job:iotime:rate_5m - job:iotime:rate_5m:avg_over_time_1h) / job:iotime:rate_5m:stddev_over_time_1h) > 3
          for: 10m
          labels:
            severity: warn
          annotations:
            summary: IO time spent (warning)

        - expr: 'rate(collectd_disk_disk_io_time_weightedio_time_total[5m])'
          record: 'job:weightiotime:rate_5m'
        - expr: 'stddev_over_time(job:weightiotime:rate_5m[1h])'
          record: 'job:weightiotime:rate_5m:stddev_over_time_1h'
        - expr: 'avg_over_time(job:weightiotime:rate_5m[1h])'
          record: 'job:weightiotime:rate_5m:avg_over_time_1h'
        - alert: weightiotime warn
          expr: >-
            (abs(job:weightiotime:rate_5m - job:weightiotime:rate_5m:avg_over_time_1h) / job:weightiotime:rate_5m:stddev_over_time_1h) > 3
          for: 10m
          labels:
            severity: warn
          annotations:
            summary: Weighted IO time spent (warning)

        - expr: 'rate(collectd_disk_disk_time_read_total[5m])'
          record: 'job:disk:time:read:rate_5m'
        - expr: 'stddev_over_time(job:disk:time:read:rate_5m[1h])'
          record: 'job:disk:time:read:rate_5m:stddev_over_time_1h'
        - expr: 'avg_over_time(job:disk:time:read:rate_5m[1h])'
          record: 'job:disk:time:read:rate_5m:avg_over_time_1h'
        - alert: disk:time:read warn
          expr: >-
            (abs(job:disk:time:read:rate_5m - job:disk:time:read:rate_5m:avg_over_time_1h) / job:disk:time:read:rate_5m:stddev_over_time_1h) >3
          for: 10m
          labels:
            severity: warn
          annotations:
            summary: IO read (warning)

        - alert: disk:time:read critical
          expr: >-
            (abs(job:disk:time:read:rate_5m - job:disk:time:read:rate_5m:avg_over_time_1h) / job:disk:time:read:rate_5m:stddev_over_time_1h) >6
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: IO read (critical)
        - expr: 'rate(collectd_disk_disk_time_write_total[5m])'
          record: 'job:disk:time:write:rate_5m'
        - expr: 'stddev_over_time(job:disk:time:write:rate_5m[1h])'
          record: 'job:disk:time:write:rate_5m:stddev_over_time_1h'
        - expr: 'avg_over_time(job:disk:time:write:rate_5m[1h])'
          record: 'job:disk:time:write:rate_5m:avg_over_time_1h'
        - alert: disk:time:write warn
          expr: >-
            (abs(job:disk:time:write:rate_5m - job:disk:time:write:rate_5m:avg_over_time_1h) / job:disk:time:write:rate_5m:stddev_over_time_1h) >3
          for: 10m
          labels:
            severity: warn
          annotations:
            summary: IO write (warning)

        - alert: disk:time:write critical
          expr: >-
            (abs(job:disk:time:write:rate_5m - job:disk:time:write:rate_5m:avg_over_time_1h) / job:disk:time:write:rate_5m:stddev_over_time_1h) >6
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: IO write (critical)
        - expr: 'rate(collectd_disk_disk_ops_read_total[5m])'
          record: 'job:disk:time:read:rate_5m'
        - expr: 'stddev_over_time(job:disk:time:read:rate_5m[1h])'
          record: 'job:disk:time:read:rate_5m:stddev_over_time_1h'
        - expr: 'avg_over_time(job:disk:time:read:rate_5m[1h])'
          record: 'job:disk:time:read:rate_5m:avg_over_time_1h'
        - alert: disk:time:read warn
          expr: >-
            (abs(job:disk:time:read:rate_5m - job:disk:time:read:rate_5m:avg_over_time_1h) / job:disk:time:read:rate_5m:stddev_over_time_1h) >3
          for: 10m
          labels:
            severity: warn
          annotations:
            summary: disk ops read (warning)

        - alert: disk:time:read critical
          expr: >-
            (abs(job:disk:time:read:rate_5m - job:disk:time:read:rate_5m:avg_over_time_1h) / job:disk:time:read:rate_5m:stddev_over_time_1h) >6
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: disk ops read (critical)
        - expr: 'rate(collectd_disk_disk_ops_write_total[5m])'
          record: 'job:disk:time:write:rate_5m'
        - expr: 'stddev_over_time(job:disk:time:write:rate_5m[1h])'
          record: 'job:disk:time:write:rate_5m:stddev_over_time_1h'
        - expr: 'avg_over_time(job:disk:time:write:rate_5m[1h])'
          record: 'job:disk:time:write:rate_5m:avg_over_time_1h'
        - alert: disk:time:write warn
          expr: >-
            (abs(job:disk:time:write:rate_5m - job:disk:time:write:rate_5m:avg_over_time_1h) / job:disk:time:write:rate_5m:stddev_over_time_1h) >3
          for: 10m
          labels:
            severity: warn
          annotations:
            summary: disk ops write (warning)

        - alert: disk:time:write critical
          expr: >-
            (abs(job:disk:time:write:rate_5m - job:disk:time:write:rate_5m:avg_over_time_1h) / job:disk:time:write:rate_5m:stddev_over_time_1h) >6
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: disk ops write (critical)
        - alert: high cpu
          labels:
            severity: warn
          annotations:
            summary: CPU usage high (warning)
          expr: >-
            sum without(plugin_instance,type_instance) (collectd_cpu_percent{type_instance=~"user|system"}) / sum without(plugin_instance,type_instance) (collectd_cpu_percent{type_instance="idle"}) > 0.5
          for: 10m
        - alert: high cpu
          labels:
            severity: critical
          annotations:
            summary: CPU usage high (critical)
          expr: >-
            sum without(plugin_instance,type_instance) (collectd_cpu_percent{type_instance=~"user|system"}) / sum without(plugin_instance,type_instance) (collectd_cpu_percent{type_instance="idle"}) > 0.7
          for: 10m
        - alert: inode usage
          labels:
            severity: warning
          annotations:
            summary: Inodes usage (warning)
          expr: >-
            sum without (endpoint,service,type_instance) (collectd_df_df_inodes{plugin_instance="root",type_instance="used"})/ (sum without (endpoint,service,type_instance) (collectd_df_df_inodes{plugin_instance="root",type_instance=~"free|used"})) > 0.6
          for: 10m
        - alert: inode usage
          labels:
            severity: critical
          annotations:
            summary: Inodes usage (critical)
          expr: >-
            sum without (endpoint,service,type_instance) (collectd_df_df_inodes{plugin_instance="root",type_instance="used"})/ (sum without (endpoint,service,type_instance) (collectd_df_df_inodes{plugin_instance="root",type_instance=~"free|used"})) > 0.8
          for: 10m
        - alert: hugepages
          labels:
            severity: warning
          annotations:
            summary: Hugepages (warning)
          expr: >-
            sum without (type_instance) (collectd_hugepages_vmpage_number{type_instance="free"})/ sum without (type_instance) (collectd_hugepages_vmpage_number) < 0.2
          for: 10m
        - alert: hugepages
          labels:
            severity: critical
          annotations:
            summary: Hugepages (warning)
          expr: >-
            sum without (type_instance) (collectd_hugepages_vmpage_number{type_instance="free"})/ sum without (type_instance) (collectd_hugepages_vmpage_number) < 0.1
          for: 10m
        - alert: load longterm
          labels:
            severity: warning
          annotations:
            summary: load longterm (warning)
          expr: >-
            (collectd_load_longterm / collectd_cpu_count) > 0.7 and (collectd_load_longterm / collectd_cpu_count) < 0.9
          for: 10m
        - alert: load longterm
          labels:
            severity: critical
          annotations:
            summary: load longterm (critical)
          expr: >-
            (collectd_load_longterm / collectd_cpu_count) >= 0.9
          for: 10m
        - alert: load midterm
          labels:
            severity: warning
          annotations:
            summary: load midterm (warning)
          expr: >-
            (collectd_load_midterm / collectd_cpu_count) > 0.7 and (collectd_load_midterm / collectd_cpu_count) < 0.9
          for: 10m
        - alert: load midterm
          labels:
            severity: critical
          annotations:
            summary: load midterm (critical)
          expr: >-
            (collectd_load_midterm / collectd_cpu_count) >= 0.9
          for: 10m
        - alert: load shortterm
          labels:
            severity: warning
          annotations:
            summary: load shortterm (warning)
          expr: >-
            (collectd_load_shortterm / collectd_cpu_count) > 0.7 and (collectd_load_shortterm / collectd_cpu_count) < 0.9
          for: 10m
        - alert: load shortterm
          labels:
            severity: critical
          annotations:
            summary: load shortterm (critical)
          expr: >-
            (collectd_load_shortterm / collectd_cpu_count) >= 0.9
          for: 10m
        - alert: memory low
          labels:
            severity: warning
          annotations:
            summary: memory low (warning)
          expr: >-
            sum without(type_instance) (collectd_memory{type_instance="used"})/sum without(type_instance) (collectd_memory) > 0.8 and sum without(type_instance) (collectd_memory{type_instance="used"})/sum without(type_instance) (collectd_memory) < 0.9
          for: 10m
        - alert: memory low
          labels:
            severity: critical
          annotations:
            summary: memory low (critical)
          expr: >-
            sum without(type_instance) (collectd_memory{type_instance="used"})/sum without(type_instance) (collectd_memory) >= 0.9
          for: 10m
