# EDITOR's NOTE: Please keep rules in this file in sync with static/samples/portworx-enterprise/k8s/prometheus/06-portworx-rules.yaml
groups:
- name: px.rules
  rules:
  # --- Portworx Volume Alerts ---
  - alert: PortworxVolumeUsageCritical
    expr: 100 * (px_volume_usage_bytes / px_volume_capacity_bytes) > 80
    for: 5m
    labels:
      issue: Portworx volume {{$labels.volumeid}} usage on {{$labels.host}} is high.
      severity: critical
    annotations:
      description: Portworx volume {{$labels.volumeid}} on {{$labels.host}} is over
        80% used for more than 10 minutes.
      summary: Portworx volume capacity is at {{$value}}% used.
  - alert: PortworxVolumeUsage
    expr: 100 * (px_volume_usage_bytes / px_volume_capacity_bytes) > 70
    for: 5m
    labels:
      issue: Portworx volume {{$labels.volumeid}} usage on {{$labels.host}} is critical.
      severity: warning
    annotations:
      description: Portworx volume {{$labels.volumeid}} on {{$labels.host}} is over
        70% used for more than 10 minutes.
      summary: Portworx volume {{$labels.volumeid}} on {{$labels.host}} is at {{$value}}%
        used.
  # --- Portworx Storage Alerts ---
  - alert: PortworxStorageUsageCritical
    expr: 100 * (1 - (px_cluster_disk_utilized_bytes / px_cluster_disk_available_bytes)) < 20
    for: 5m
    labels:
      issue: Portworx storage {{$labels.volumeid}} usage on {{$labels.host}} is high.
      severity: critical
    annotations:
      description: Portworx storage {{$labels.volumeid}} on {{$labels.host}} is over
        80% used for more than 10 minutes.
      summary: Portworx storage capacity is at {{$value}}% used.
  - alert: PortworxStorageUsage
    expr: 100 * (1 - (px_cluster_disk_utilized_bytes / px_cluster_disk_available_bytes)) < 30
    for: 5m
    labels:
      issue: Portworx storage {{$labels.volumeid}} usage on {{$labels.host}} is critical.
      severity: warning
    annotations:
      description: Portworx storage {{$labels.volumeid}} on {{$labels.host}} is over
        70% used for more than 10 minutes.
      summary: Portworx storage {{$labels.volumeid}} on {{$labels.host}} is at {{$value}}%
        used.
  - alert: PortworxStorageWillFill
    expr: (100 * (1 - (px_cluster_disk_utilized_bytes / px_cluster_disk_available_bytes))) < 30
      and predict_linear(px_cluster_disk_available_bytes[1h], 14 * 86400) < 0
    for: 10m
    labels:
      issue: Portworx storage {{$labels.volumeid}} on {{$labels.host}} is predicted
        to fill within 2 weeks.
      severity: warning
    annotations:
      description: Portworx storage {{$labels.volumeid}} on {{$labels.host}} is over
        70% full and has been predicted to fill within 2 weeks for more than 10 minutes.
      summary: Portworx storage {{$labels.volumeid}} on {{$labels.host}} is over 70%
        full and is predicted to fill within 2 weeks.
  # --- Portworx Cluster Alerts ---
  - alert: PortworxStorageNodeDown
    expr: max(px_cluster_status_nodes_storage_down) > 0
    for: 5m
    labels:
      issue: Portworx Storage Node is Offline.
      severity: critical
    annotations:
      description: Portworx Storage Node has been offline for more than 5 minutes.
      summary: Portworx Storage Node is Offline.
  - alert: PortworxQuorumUnhealthy
    expr: max(px_cluster_status_cluster_quorum) > 1
    for: 5m
    labels:
      issue: Portworx Quorum Unhealthy.
      severity: critical
    annotations:
      description: Portworx cluster Quorum Unhealthy for more than 5 minutes.
      summary: Portworx Quorum Unhealthy.
  - alert: PortworxMemberDown
    expr: (max(px_cluster_status_cluster_size) - count(px_cluster_status_cluster_size)) > 0
    for: 5m
    labels:
      issue: Portworx cluster member(s) is(are) down.
      severity: critical
    annotations:
      description: 'Portworx cluster member(s) has(have) been down for
        more than 5 minutes.'
      summary: Portworx cluster member(s) is(are) down.
  # --- Portworx License Alerts ---
  - alert: PortworxLicenseExpiry
    # Tuneable license-expiry alert, reported by each node individually, works for both regular and floating licenses.
    # - note: if #days drops into negative, the license has already expired
    expr: px_node_status_license_expiry < 5
    labels:
      issue: Portworx license (or license lease) expiring.
      severity: warning
    annotations:
      summary: Portworx license (or license lease) expiring in {{$value}} days.
      description: Portworx node {{$labels.node}} of cluster {{$labels.cluster}} reports
        its license (or license lease) expiring in {{$value}} days.
  - alert: PortworxLicenseServerDown
    expr: px_alerts_licenseserverdown > 0
    labels:
      severity: warning
      issue: Portworx license server is unreachable
    annotations:
      summary: Portworx instance {{$labels.instance}} is not able to reach the license server.
      description: Portworx license server is unreachable. Please check license
        server health and connection to license server.
  - alert: PortworxLicenseSetupError
    expr: px_alerts_floatinglicensesetuperror > 0
    labels:
      issue: Error setting up Portworx license
      severity: critical
    annotations:
      summary: Failed to set up Portworx license on {{$labels.instance}}.
      description: Failed to set up Portworx license on {{$labels.instance}}.
        Please check licenses and/or license server.
  # --- Miscellanies Alerts ---
  - alert: PXBackupError
    expr: px_backup_stats_status == 2
    labels:
      issue: Cloudsnap backup error
      severity: warning
    annotations:
      description: Failed to take backup for volume {{$labels.volumename}} with error {{$labels.error_string}}.
      summary: Failed to take backup for volume {{$labels.volumename}}.
