diff --git a/docs/tasks/NOOK-152/README.md b/docs/tasks/NOOK-152/README.md new file mode 100644 index 00000000..10ab6183 --- /dev/null +++ b/docs/tasks/NOOK-152/README.md @@ -0,0 +1,52 @@ +# NOOK-152 Grafana DatasourceError 반복 알림과 로그 통계 불일치 수정 + +## 목적 + +일시적인 Prometheus/Loki 조회 실패가 모든 업무 Alert의 `DatasourceError`로 증폭되는 문제를 줄이고, +로그 대시보드의 ERROR/WARN 통계를 실제 조회 범위와 일치시킵니다. + +## 원인 + +- Grafana-managed Alert의 `execErrState: Error`는 조회 실패마다 원래 Alert와 독립적인 + `DatasourceError`를 생성합니다. +- 기존 정책은 모든 `DatasourceError`를 같은 이름으로 묶어 1시간마다 반복 전송했습니다. +- datasource provisioning이 Grafana 시작 시 기존 Prometheus/Loki datasource를 삭제한 뒤 다시 + 생성해, 시작 구간의 조회 실패 가능성을 키웠습니다. +- 로그 stat은 `count_over_time(...[$__range])`를 range 쿼리로 실행해 조회 종료 시점에는 데이터가 + 없어도 과거의 마지막 non-null 값을 표시했습니다. + +## 범위 + +- ERROR/WARN stat을 instant 쿼리로 변경합니다. +- 업무 Alert의 실행 오류 상태를 Grafana provisioning enum인 `KeepLast`로 변경합니다. +- Prometheus/Loki 연결 상태를 확인하는 전용 Alert를 추가합니다. +- `DatasourceError`를 datasource와 원본 규칙별로 그룹화하고 최초 알림을 2분 지연하며 반복 주기를 + 6시간으로 분리합니다. +- datasource 삭제 후 재생성을 제거합니다. +- Grafana 시작 전 Prometheus/Loki readiness를 최대 60초 확인합니다. 제한 시간 뒤에는 진단을 위해 + Grafana를 시작합니다. + +## 제외 범위 + +- 애플리케이션 로그 레벨 또는 로그 포맷 변경 +- dev API 기능/API 계약 변경 +- live 환경 Alert 임계치 도입 + +## 성공 기준 + +- 조회 범위에 ERROR/WARN 로그가 없으면 stat이 0을 표시합니다. +- 일시적 datasource 조회 실패가 모든 업무 Alert의 `DatasourceError`로 확산되지 않습니다. +- 2분 이상 지속되는 Prometheus/Loki 실행 오류는 전용 규칙의 `DatasourceError`로 식별됩니다. +- alert provisioning YAML, dashboard JSON, Docker Compose 구성이 유효합니다. + +## 검증 + +```shell +ruby -e 'require "yaml"; Dir["ops/**/*.yml"].each { |file| YAML.load_file(file) }' +jq empty ops/monitoring/grafana/dashboards/*.json +docker compose -f ops/monitoring/docker-compose.yml config +./gradlew check +``` + +ops VM 반영 후 Grafana rules API에서 모든 규칙의 health/state를 확인하고, Loki와 Prometheus의 +`vector(1)` 쿼리 및 로그 stat의 instant 쿼리 결과를 직접 비교합니다. diff --git a/ops/monitoring/docker-compose.yml b/ops/monitoring/docker-compose.yml index 9ecf0447..cad34e25 100644 --- a/ops/monitoring/docker-compose.yml +++ b/ops/monitoring/docker-compose.yml @@ -29,6 +29,9 @@ services: grafana: image: grafana/grafana:latest container_name: nook-ops-grafana + entrypoint: + - /bin/sh + - /etc/grafana/startup/wait-for-datasources.sh depends_on: - prometheus - loki @@ -41,6 +44,7 @@ services: - "127.0.0.1:3000:3000" volumes: - grafana-data:/var/lib/grafana + - ./grafana/startup:/etc/grafana/startup:ro - ./grafana/provisioning:/etc/grafana/provisioning:ro - ./grafana/dashboards:/var/lib/grafana/dashboards:ro restart: unless-stopped diff --git a/ops/monitoring/grafana/dashboards/nook-dev-logs.json b/ops/monitoring/grafana/dashboards/nook-dev-logs.json index 669a75a7..002871b9 100644 --- a/ops/monitoring/grafana/dashboards/nook-dev-logs.json +++ b/ops/monitoring/grafana/dashboards/nook-dev-logs.json @@ -158,8 +158,8 @@ "uid": "loki" }, "editorMode": "code", - "expr": "sum(count_over_time({env=\"dev\", job=\"nook-api\", level=\"ERROR\"}[$__range]))", - "queryType": "range", + "expr": "sum(count_over_time({env=\"dev\", job=\"nook-api\", level=\"ERROR\"}[$__range])) or on() vector(0)", + "queryType": "instant", "refId": "A" } ], @@ -220,8 +220,8 @@ "uid": "loki" }, "editorMode": "code", - "expr": "sum(count_over_time({env=\"dev\", job=\"nook-api\", level=\"WARN\"}[$__range]))", - "queryType": "range", + "expr": "sum(count_over_time({env=\"dev\", job=\"nook-api\", level=\"WARN\"}[$__range])) or on() vector(0)", + "queryType": "instant", "refId": "A" } ], diff --git a/ops/monitoring/grafana/provisioning/alerting/nook-datasource-alerts.yml b/ops/monitoring/grafana/provisioning/alerting/nook-datasource-alerts.yml new file mode 100644 index 00000000..02e8bf5a --- /dev/null +++ b/ops/monitoring/grafana/provisioning/alerting/nook-datasource-alerts.yml @@ -0,0 +1,155 @@ +apiVersion: 1 + +groups: + - orgId: 1 + name: nook-datasource-alerts + folder: Nook + interval: 1m + rules: + - uid: nook_prometheus_datasource_unavailable + title: Prometheus datasource unavailable + condition: C + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: + from: 300 + to: 0 + model: + datasource: + type: prometheus + uid: prometheus + editorMode: code + expr: vector(1) + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + refId: A + - refId: B + datasourceUid: __expr__ + relativeTimeRange: + from: 300 + to: 0 + model: + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + datasourceUid: __expr__ + relativeTimeRange: + from: 300 + to: 0 + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: Alerting + execErrState: Error + for: 2m + annotations: + summary: Grafana에서 Prometheus datasource에 연결할 수 없습니다. + description: Prometheus 조회가 2분 이상 실패했습니다. Grafana와 Prometheus의 컨테이너 상태 및 Docker DNS를 확인하세요. + labels: + env: ops + service: prometheus + severity: warning + isPaused: false + + - uid: nook_loki_datasource_unavailable + title: Loki datasource unavailable + condition: C + data: + - refId: A + datasourceUid: loki + relativeTimeRange: + from: 300 + to: 0 + model: + datasource: + type: loki + uid: loki + editorMode: code + expr: vector(1) + intervalMs: 1000 + maxDataPoints: 43200 + queryType: range + refId: A + - refId: B + datasourceUid: __expr__ + relativeTimeRange: + from: 300 + to: 0 + model: + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + datasourceUid: __expr__ + relativeTimeRange: + from: 300 + to: 0 + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: Alerting + execErrState: Error + for: 2m + annotations: + summary: Grafana에서 Loki datasource에 연결할 수 없습니다. + description: Loki 조회가 2분 이상 실패했습니다. Grafana와 Loki의 컨테이너 상태 및 Docker DNS를 확인하세요. + labels: + env: ops + service: loki + severity: warning + isPaused: false diff --git a/ops/monitoring/grafana/provisioning/alerting/nook-dev-api-errors.yml b/ops/monitoring/grafana/provisioning/alerting/nook-dev-api-errors.yml index d096ae3f..b8dab37e 100644 --- a/ops/monitoring/grafana/provisioning/alerting/nook-dev-api-errors.yml +++ b/ops/monitoring/grafana/provisioning/alerting/nook-dev-api-errors.yml @@ -69,7 +69,7 @@ groups: refId: C type: threshold noDataState: OK - execErrState: Error + execErrState: KeepLast for: 1m annotations: summary: dev API에서 ERROR 로그가 감지됐습니다. diff --git a/ops/monitoring/grafana/provisioning/alerting/nook-dev-core-alerts.yml b/ops/monitoring/grafana/provisioning/alerting/nook-dev-core-alerts.yml index d6948142..ac279f05 100644 --- a/ops/monitoring/grafana/provisioning/alerting/nook-dev-core-alerts.yml +++ b/ops/monitoring/grafana/provisioning/alerting/nook-dev-core-alerts.yml @@ -64,7 +64,7 @@ groups: refId: C type: threshold noDataState: Alerting - execErrState: Error + execErrState: KeepLast for: 2m annotations: summary: dev API target이 응답하지 않습니다. @@ -133,7 +133,7 @@ groups: refId: C type: threshold noDataState: Alerting - execErrState: Error + execErrState: KeepLast for: 5m annotations: summary: dev MySQL exporter target이 응답하지 않습니다. @@ -202,7 +202,7 @@ groups: refId: C type: threshold noDataState: Alerting - execErrState: Error + execErrState: KeepLast for: 5m annotations: summary: dev node exporter target이 응답하지 않습니다. @@ -271,7 +271,7 @@ groups: refId: C type: threshold noDataState: OK - execErrState: Error + execErrState: KeepLast for: 5m annotations: summary: dev API의 HTTP 5xx 응답이 증가했습니다. @@ -340,7 +340,7 @@ groups: refId: C type: threshold noDataState: OK - execErrState: Error + execErrState: KeepLast for: 10m annotations: summary: dev API의 HTTP p95 latency가 높습니다. @@ -409,7 +409,7 @@ groups: refId: C type: threshold noDataState: OK - execErrState: Error + execErrState: KeepLast for: 15m annotations: summary: dev VM 루트 디스크 사용률이 80%를 넘었습니다. @@ -478,7 +478,7 @@ groups: refId: C type: threshold noDataState: OK - execErrState: Error + execErrState: KeepLast for: 5m annotations: summary: dev VM 루트 디스크 사용률이 90%를 넘었습니다. @@ -547,7 +547,7 @@ groups: refId: C type: threshold noDataState: OK - execErrState: Error + execErrState: KeepLast for: 10m annotations: summary: dev VM 메모리 사용률이 90%를 넘었습니다. @@ -616,7 +616,7 @@ groups: refId: C type: threshold noDataState: OK - execErrState: Error + execErrState: KeepLast for: 3m annotations: summary: dev API의 DB connection pool이 고갈되고 있습니다. @@ -685,7 +685,7 @@ groups: refId: C type: threshold noDataState: OK - execErrState: Error + execErrState: KeepLast for: 0s annotations: summary: dev API에서 DB connection 획득 timeout이 발생했습니다. diff --git a/ops/monitoring/grafana/provisioning/alerting/notification-policies.yml b/ops/monitoring/grafana/provisioning/alerting/notification-policies.yml index 78c5830c..699513f5 100644 --- a/ops/monitoring/grafana/provisioning/alerting/notification-policies.yml +++ b/ops/monitoring/grafana/provisioning/alerting/notification-policies.yml @@ -9,3 +9,15 @@ policies: group_wait: 30s group_interval: 5m repeat_interval: 1h + routes: + - receiver: slack-nook-alerts-v2 + object_matchers: + - - alertname + - = + - DatasourceError + group_by: + - datasource_uid + - rulename + group_wait: 2m + group_interval: 10m + repeat_interval: 6h diff --git a/ops/monitoring/grafana/provisioning/datasources/datasources.yml b/ops/monitoring/grafana/provisioning/datasources/datasources.yml index ddc6876f..237a2297 100644 --- a/ops/monitoring/grafana/provisioning/datasources/datasources.yml +++ b/ops/monitoring/grafana/provisioning/datasources/datasources.yml @@ -1,11 +1,5 @@ apiVersion: 1 -deleteDatasources: - - name: Prometheus - orgId: 1 - - name: Loki - orgId: 1 - datasources: - name: Prometheus type: prometheus diff --git a/ops/monitoring/grafana/startup/wait-for-datasources.sh b/ops/monitoring/grafana/startup/wait-for-datasources.sh new file mode 100755 index 00000000..29a70c56 --- /dev/null +++ b/ops/monitoring/grafana/startup/wait-for-datasources.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -eu + +max_attempts=30 +attempt=1 + +while [ "$attempt" -le "$max_attempts" ]; do + if wget -q -T 2 -O /dev/null http://prometheus:9090/-/ready \ + && wget -q -T 2 -O /dev/null http://loki:3100/ready; then + exec /run.sh + fi + + echo "Waiting for Prometheus and Loki readiness (${attempt}/${max_attempts})" >&2 + attempt=$((attempt + 1)) + sleep 2 +done + +echo "Datasource readiness wait timed out; starting Grafana for diagnostics" >&2 +exec /run.sh