apiVersion: 1
groups:
  - orgId: 1
    name: system-monitor
    folder: System Monitor
    interval: 30s
    rules:
      - uid: mem_avail_low_crit
        title: mem_avail_low_crit
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 8', instant: true, refId: A}
        for: 1m
        noDataState: OK
        execErrState: Error
        labels: {rule: mem_avail_low, stack: system-monitor, remediation: none, severity: critical, notify: desktop}
        annotations: {summary: 'Memory available below 8% on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '2'}
      - uid: mem_avail_low_warn
        title: mem_avail_low_warn
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 15', instant: true, refId: A}
        for: 1m
        noDataState: OK
        execErrState: Error
        labels: {rule: mem_avail_low, stack: system-monitor, remediation: none, severity: warning}
        annotations: {summary: 'Memory available below 15% on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '2'}
      - uid: mem_exhaustion_eta_crit
        title: mem_exhaustion_eta_crit
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_mem_exhaustion_eta_seconds < 90', instant: true, refId: A}
        for: 0s
        noDataState: OK
        execErrState: Error
        labels: {rule: mem_exhaustion_eta, stack: system-monitor, remediation: kill, severity: critical, notify: desktop}
        annotations: {summary: 'Memory exhaustion ETA below 90s on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '14'}
      - uid: mem_exhaustion_eta_warn
        title: mem_exhaustion_eta_warn
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_mem_exhaustion_eta_seconds < 360', instant: true, refId: A}
        for: 0s
        noDataState: OK
        execErrState: Error
        labels: {rule: mem_exhaustion_eta, stack: system-monitor, remediation: kill, severity: warning, notify: desktop}
        annotations: {summary: 'Memory exhaustion ETA below 360s on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '14'}
      - uid: mem_psi_high
        title: mem_psi_high
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'rate(node_pressure_memory_stalled_seconds_total[1m]) > 0.20', instant: true, refId: A}
        for: 1m
        noDataState: OK
        execErrState: Error
        labels: {rule: mem_psi_high, stack: system-monitor, remediation: none, severity: critical}
        annotations: {summary: 'Memory PSI stalled above 20% on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '13'}
      - uid: io_pressure_thrash
        title: io_pressure_thrash
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'rate(node_pressure_io_stalled_seconds_total[5m]) > 0.30', instant: true, refId: A}
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: {rule: io_pressure_thrash, stack: system-monitor, remediation: none, severity: warning}
        annotations: {summary: 'IO PSI stalled above 30% on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '13'}
      - uid: swap_near_full
        title: swap_near_full
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_memory_SwapTotal_bytes > 0 and node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes * 100 < 10', instant: true, refId: A}
        for: 2m
        noDataState: OK
        execErrState: Error
        labels: {rule: swap_near_full, stack: system-monitor, remediation: none, severity: warning}
        annotations: {summary: 'Swap free below 10% on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '3'}
      - uid: disk_fill_predict_crit
        title: disk_fill_predict_crit
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 7200, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|overlay|squashfs"}[1h], 3600) < 0', instant: true, refId: A}
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: {rule: disk_fill_predict, stack: system-monitor, remediation: none, severity: critical, notify: desktop}
        annotations: {summary: 'Filesystem {{ $labels.mountpoint }} will fill within 1h on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '22'}
      - uid: disk_fill_predict_warn
        title: disk_fill_predict_warn
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 7200, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|overlay|squashfs"}[1h], 14400) < 0', instant: true, refId: A}
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: {rule: disk_fill_predict, stack: system-monitor, remediation: none, severity: warning}
        annotations: {summary: 'Filesystem {{ $labels.mountpoint }} will fill within 4h on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '22'}
      - uid: disk_used_high
        title: disk_used_high
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: '(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|ramfs|overlay|squashfs"}) * 100 > 90', instant: true, refId: A}
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: {rule: disk_used_high, stack: system-monitor, remediation: none, severity: warning}
        annotations: {summary: 'Filesystem {{ $labels.mountpoint }} above 90% on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '20'}
      - uid: inodes_low
        title: inodes_low
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_filesystem_files{fstype!~"tmpfs|ramfs|overlay|squashfs"} > 0 and node_filesystem_files_free{fstype!~"tmpfs|ramfs|overlay|squashfs"} / node_filesystem_files{fstype!~"tmpfs|ramfs|overlay|squashfs"} * 100 < 10', instant: true, refId: A}
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: {rule: inodes_low, stack: system-monitor, remediation: none, severity: warning}
        annotations: {summary: 'Filesystem {{ $labels.mountpoint }} inode free below 10% on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '21'}
      - uid: tmpfs_junk_crit
        title: tmpfs_junk_crit
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_tmp_used_percent > 95 or node_tmp_inodes_percent > 95', instant: true, refId: A}
        for: 1m
        noDataState: OK
        execErrState: Error
        labels: {rule: tmpfs_junk, stack: system-monitor, remediation: clean, severity: critical, notify: desktop}
        annotations: {summary: '/tmp usage or inodes above 95% on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '25'}
      - uid: tmpfs_junk_warn
        title: tmpfs_junk_warn
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_tmp_used_percent > 85 or node_tmp_inodes_percent > 85', instant: true, refId: A}
        for: 1m
        noDataState: OK
        execErrState: Error
        labels: {rule: tmpfs_junk, stack: system-monitor, remediation: clean, severity: warning}
        annotations: {summary: '/tmp usage or inodes above 85% on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '25'}
      - uid: cpu_runaway_nonbuild
        title: cpu_runaway_nonbuild
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_cpu_nonbuild_busy_percent > 85', instant: true, refId: A}
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: {rule: cpu_runaway_nonbuild, stack: system-monitor, remediation: kill, severity: warning}
        annotations: {summary: 'Non-build CPU group above 85% on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '17'}
      - uid: fd_exhaustion
        title: fd_exhaustion
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_filefd_allocated / node_filefd_maximum * 100 > 80', instant: true, refId: A}
        for: 2m
        noDataState: OK
        execErrState: Error
        labels: {rule: fd_exhaustion, stack: system-monitor, remediation: none, severity: warning}
        annotations: {summary: 'File descriptor table above 80% on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '29'}
      - uid: proc_explosion
        title: proc_explosion
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'sum(node_processes_threads) > 12000', instant: true, refId: A}
        for: 2m
        noDataState: OK
        execErrState: Error
        labels: {rule: proc_explosion, stack: system-monitor, remediation: none, severity: warning}
        annotations: {summary: 'Process threads above 12000', __dashboardUid__: system-monitor-overview, __panelId__: '28'}
      - uid: unit_failed
        title: unit_failed
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_systemd_unit_state{state="failed"} == 1', instant: true, refId: A}
        for: 0s
        noDataState: OK
        execErrState: Error
        labels: {rule: unit_failed, stack: system-monitor, remediation: none, severity: warning}
        annotations: {summary: 'systemd unit {{ $labels.name }} failed on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '9'}
      - uid: oom_kill
        title: oom_kill
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'increase(node_vmstat_oom_kill[5m]) > 0', instant: true, refId: A}
        for: 0s
        noDataState: OK
        execErrState: Error
        labels: {rule: oom_kill, stack: system-monitor, remediation: none, severity: critical, notify: desktop}
        annotations: {summary: 'OOM kill observed on {{ $labels.instance }}'}
      - uid: temp_high_crit
        title: temp_high_crit
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'max_over_time(node_thermal_zone_temp{type="x86_pkg_temp"}[5m]) > 99', instant: true, refId: A}
        for: 2m
        noDataState: OK
        execErrState: Error
        labels: {rule: temp_high, stack: system-monitor, remediation: none, severity: critical, notify: desktop}
        annotations: {summary: 'CPU package sustained above 99C on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '8'}
      - uid: temp_high_warn
        title: temp_high_warn
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'max_over_time(node_thermal_zone_temp{type="x86_pkg_temp"}[5m]) > 95', instant: true, refId: A}
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: {rule: temp_high, stack: system-monitor, remediation: none, severity: warning}
        annotations: {summary: 'CPU package sustained above 95C on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '8'}
      - uid: smart_unhealthy
        title: smart_unhealthy
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_smartmon_device_smart_healthy == 0', instant: true, refId: A}
        for: 0s
        noDataState: OK
        execErrState: Error
        labels: {rule: smart_unhealthy, stack: system-monitor, remediation: none, severity: critical, notify: desktop}
        annotations: {summary: 'SMART health failed for {{ $labels.device }} on {{ $labels.instance }}'}
      - uid: smart_realloc
        title: smart_realloc
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_smartmon_reallocated_sector_ct_raw_value > 0', instant: true, refId: A}
        for: 0s
        noDataState: OK
        execErrState: Error
        labels: {rule: smart_realloc, stack: system-monitor, remediation: none, severity: warning}
        annotations: {summary: 'SMART reallocated sectors for {{ $labels.device }} on {{ $labels.instance }}'}
      - uid: backup_stale
        title: backup_stale
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'node_backup_age_seconds > 604800', instant: true, refId: A}
        for: 0s
        noDataState: OK
        execErrState: Error
        labels: {rule: backup_stale, stack: system-monitor, remediation: none, severity: warning}
        annotations: {summary: 'Newest configured backup is older than 7 days on {{ $labels.instance }}'}
      - uid: monitor_target_down
        title: monitor_target_down
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'up{job=~"node|prometheus"} == 0', instant: true, refId: A}
        for: 5m
        noDataState: Alerting
        execErrState: Error
        labels: {rule: monitor_degraded, stack: system-monitor, remediation: none, severity: critical, notify: desktop}
        annotations: {summary: 'Monitoring pipeline degraded: {{ $labels.job }} target is DOWN — metric coverage (incl. OOM protection) is blind', __dashboardUid__: system-monitor-overview, __panelId__: '1'}
      - uid: monitor_oom_feed_stale
        title: monitor_oom_feed_stale
        condition: A
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: __PROMETHEUS_DS_UID__
            model: {expr: 'time() - node_textfile_mtime_seconds{file="/var/lib/node_exporter/textfile/mem_exhaustion.prom"} > 300', instant: true, refId: A}
        for: 2m
        noDataState: Alerting
        execErrState: Error
        labels: {rule: monitor_degraded, stack: system-monitor, remediation: none, severity: critical, notify: desktop}
        annotations: {summary: 'OOM early-warning feed is stale (mem_exhaustion producer not updating) — exhaustion ETA is blind on {{ $labels.instance }}', __dashboardUid__: system-monitor-overview, __panelId__: '14'}
