Loki —— 分布式系统轻量级日志收集器

• 搭建loki

#!/bin/bash
cd /data/server;
mkdir -p loki;
cd loki;
# 下载最新版本Loki（20220620）
$ curl -O -L "https://github.com/grafana/loki/releases/download/v2.8.2/loki-linux-amd64.zip"
# 解压
$ unzip "loki-linux-amd64.zip"
$ chmod a+x "loki-linux-amd64"
# 创建loki配置
touch loki-config-test.yaml;
# 修改 loki-config-test.yaml（请参照下面loki-config-test.yaml配置）;
# 启动 Loki 服务：
nohup ./loki-linux-amd64 -config.file=loki-hw-test-config.yaml -config.expand-env=true > start-loki.log &;
tail -100f start-loki.log;

• loki-config-test.yaml配置

# 以下配置应用到HW-TEST环境
auth_enabled: false

server:
  http_listen_port: 3100

ingester:
  lifecycler:
    address: 127.0.0.1
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 5m
  chunk_retain_period: 30s
  max_transfer_retries: 0

schema_config:
  configs:
    - from: 2021-01-01
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

storage_config:
  boltdb_shipper:
    active_index_directory: /data/loki/boltdb-shipper-active
    cache_location: /data/loki/boltdb-shipper-cache
    cache_ttl: 24h
    shared_store: filesystem
  filesystem:
    directory: /data/logs/loki

compactor:
  working_directory: /data/loki/boltdb-shipper-compactor
  compaction_interval: 1h

limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h

chunk_store_config:
  max_look_back_period: 0s

table_manager:
  retention_deletes_enabled: false
  retention_period: 0s

ruler:
  storage:
    type: local
    local:
      directory: /data/rules
  alertmanager_url: http://localhost:9093
  rule_path: /data/rules/*.yaml

done.

• grafana配置loki

  • Add data source
  • 选择Loki
  • 输入Loki的host:port
  • Save & Test 即可

• promtail安装（作为日志上报组件运行在应用节点上，每个需要收集日志的节点都要配置）

#!/bin/bash
#-------------------------------------------------#
# 脚本主要功能说明: promtail安装
#-------------------------------------------------#
mkdir -p /data/server/promtail;
cd /data/server/promtail;
curl -O -L https://github.com/grafana/loki/releases/download/v2.8.2/promtail-linux-amd64.zip;
unzip promtail-linux-amd64.zip;
chmod a+x promtail-linux-amd64;

# 配置
wget https://raw.githubusercontent.com/grafana/loki/main/clients/cmd/promtail/promtail-local-config.yaml;
mv promtail-local-config.yaml promtail-hw-test-config.yaml;
echo "Promtail 安装成功！"

• promtail配置(promtail-local-config.yaml)

# 服务器配置
server:
  # HTTP监听端口
  http_listen_port: 9080
  # GRPC监听端口，0表示由系统分配
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

# Loki客户端配置
clients:
  # Loki服务器的URL（3100是默认端口）
  - url: http://[Loki安装的节点]:3100/loki/api/v1/push

# 抓取配置
scrape_configs:
- job_name: system
  # 静态配置
  static_configs:
  - targets:
      # 目标，这里是localhost（本机）
      - localhost
    labels:
      # 标签（后续在grafana中可以查询）
      job: varlogs
      # 需要抓取的文件路径（采集所有后缀为-info.log的日志）
      __path__: /data/logs/*-info.log

• 启动promtail，并指定配置

nohup ./promtail-linux-amd64 -config.file=promtail-hw-test-config.yaml -config.expand-env=true > promtail-start.log &