{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "Isis generation pipeline: GPU utilization, model load time, inference latency by model, VRAM usage, queue wait time.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "liveNow": false, "panels": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "GPU utilization per device (DCGM exporter).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true }, "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 85 }, { "color": "red", "value": 95 } ] }, "unit": "percent" } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "id": 1, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$gpu_host\"}", "legendFormat": "{{Hostname}} gpu{{gpu}}", "range": true, "refId": "A" } ], "title": "GPU Utilization per Device", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "VRAM used per GPU (MiB).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "mbytes" } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "id": 2, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$gpu_host\"}", "legendFormat": "{{Hostname}} gpu{{gpu}}", "range": true, "refId": "A" } ], "title": "VRAM Used per GPU", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Inference latency percentiles per model.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 5000 }, { "color": "red", "value": 10000 } ] }, "unit": "ms" } }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 8 }, "id": 3, "options": { "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum by (le, model) (rate(isis_inference_latency_ms_bucket{model=~\"$model\"}[5m])))", "legendFormat": "{{model}} p50", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, model) (rate(isis_inference_latency_ms_bucket{model=~\"$model\"}[5m])))", "legendFormat": "{{model}} p95", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le, model) (rate(isis_inference_latency_ms_bucket{model=~\"$model\"}[5m])))", "legendFormat": "{{model}} p99", "range": true, "refId": "C" } ], "title": "Inference Latency by Model", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Model load time (seconds) by model.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "fillOpacity": 80, "gradientMode": "none", "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, "id": 4, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, model) (rate(isis_model_load_seconds_bucket{model=~\"$model\"}[30m])))", "legendFormat": "{{model}} p95", "range": true, "refId": "A" } ], "title": "Model Load Time p95", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Queue wait time (time between submit and start) per model.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 }, "id": 5, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, model) (rate(isis_queue_wait_seconds_bucket{model=~\"$model\"}[5m])))", "legendFormat": "{{model}} p95", "range": true, "refId": "A" } ], "title": "Queue Wait Time p95", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "GPU power draw (watts) per device.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "watt" } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 25 }, "id": 6, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$gpu_host\"}", "legendFormat": "{{Hostname}} gpu{{gpu}}", "range": true, "refId": "A" } ], "title": "GPU Power Draw", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "GPU SM temperature in Celsius.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 80 }, { "color": "red", "value": 90 } ] }, "unit": "celsius" } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 25 }, "id": 7, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$gpu_host\"}", "legendFormat": "{{Hostname}} gpu{{gpu}}", "range": true, "refId": "A" } ], "title": "GPU Temperature", "type": "timeseries" } ], "refresh": "30s", "schemaVersion": 39, "tags": ["oshun", "isis", "gpu", "inference"], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus", "value": "prometheus" }, "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": ".+", "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", "hide": 0, "includeAll": true, "label": "GPU Host", "multi": true, "name": "gpu_host", "options": [], "query": { "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" }, { "allValue": ".+", "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(isis_inference_latency_ms_bucket, model)", "hide": 0, "includeAll": true, "label": "Model", "multi": true, "name": "model", "options": [], "query": { "query": "label_values(isis_inference_latency_ms_bucket, model)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Isis Generation Pipeline", "uid": "isis-generation-pipeline", "version": 1, "weekStart": "" }