CentOS 搭建Graylog集群详解

 4. 多节点集群配置安装

  ① MongoDB集群配置:

  修改所有mongdb节点的配置文件/etc/mongod.conf,添加集群replication信息replSetName: rs0,并重启服务。

CentOS 搭建Graylog集群详解CentOS 搭建Graylog集群详解

# cat /etc/mongod.conf 
# mongod.conf

# for documentation of all options, see:
#   http://docs.mongodb.org/manual/reference/configuration-options/

# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  path: /var/log/mongodb/mongod.log

# Where and how to store data.
storage:
  dbPath: /var/lib/mongo
  journal:
    enabled: true
#  engine:
#  mmapv1:
#  wiredTiger:

# how the process runs
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile

# network interfaces
net:
  port: 27017
#  bindIp: 127.0.0.1  # Listen to local interface only, comment to listen on all interfaces.


#security:

#operationProfiling:

replication:
  replSetName: rs0
#sharding:

## Enterprise-Only Options

#auditLog:

#snmp:

mongod.conf

$ sudo systemctl restart mongod.service

  在集群其中一个节点,启动mongo命令行:

$ mongo

  初始化mongodb集群,使用本机hostname或IP加端口:

$ rs.initiate( {
   _id : "rs0",
   members: [ { _id : 0, host : "mongodb0.example.net:27017" } ]
})

  验证集群配置:

CentOS 搭建Graylog集群详解CentOS 搭建Graylog集群详解

$ rs.conf()

{
   "_id" : "rs0",
   "version" : 1,
   "protocolVersion" : NumberLong(1),
   "members" : [
      {
         "_id" : 0,
         "host" : "mongodb0.example.net:27017",
         "arbiterOnly" : false,
         "buildIndexes" : true,
         "hidden" : false,
         "priority" : 1,
         "tags" : {

         },
         "slaveDelay" : NumberLong(0),
         "votes" : 1
      }
   ],
   "settings" : {
      "chainingAllowed" : true,
      "heartbeatIntervalMillis" : 2000,
      "heartbeatTimeoutSecs" : 10,
      "electionTimeoutMillis" : 10000,
      "catchUpTimeoutMillis" : 2000,
      "getLastErrorModes" : {

      },
      "getLastErrorDefaults" : {
         "w" : 1,
         "wtimeout" : 0
      },
      "replicaSetId" : ObjectId("585ab9df685f726db2c6a840")
   }
}

rs.conf()

  将其他节点加入集群,并查看集群配置:

rs0:PRIMARY> rs.add("mongodb1.example.net")
rs0:PRIMARY> rs.add("mongodb2.example.net")
rs0:PRIMARY> rs.status()

  创建graylog数据库,并添加graylog用户,赋予readWrite和dbAdmin权限:

rs0:PRIMARY> use graylog
switched to db graylog
rs0:PRIMARY> db.createUser( {
    user: "graylog",
     pwd: "75PN76Db66En",
     roles: [ { role: "readWrite", db: "graylog" } ]
   });
rs0:PRIMARY> db.grantRolesToUser( "graylog" , [ { role: "dbAdmin", db: "graylog" } ])
rs0:PRIMARY> show users
rs0:PRIMARY> db.auth("graylog","75sdfsdsdfn"

  ② Elasticsearch 集群配置:

  修改elasticsearch配置文件并重启服务:

# cat /etc/elasticsearch/elasticsearch.yml | grep cluster.name
cluster.name: graylog
# cat /etc/elasticsearch/elasticsearch.yml | grep discovery.zen.ping
discovery.zen.ping.unicast.hosts: ["10.2.2.41", "10.2.2.43"]
# cat /etc/elasticsearch/elasticsearch.yml | grep network.host
network.host: 10.2.2.42

  ③ graylog集群配置

  graylog master节点修改配置server.conf 中 is_master true,其他节点为false,同时rest_listen_uri以及rest_transport_uri必须可以被集群中的其他节点连通。

  修改mongodb连接配置:

# cat /etc/graylog/server/server.conf|grep mongodb_uri
mongodb_uri = mongodb://graylog:75PN76Db66En@10.2.2.41:27017,10.2.2.42:27017,10.2.2.43:27017/graylog?replicaSet=rs0

  修改elasticsearch连接配置:

# cat /etc/graylog/server/server.conf|grep elasticsearch_hosts
elasticsearch_hosts = http://grayloguser:3KKLg8294CE0@10.2.2.41:9200,http://grayloguser:3KKLg8294CE0@10.2.2.42:9200,http://grayloguser:3KKLg8294CE0@10.2.2.43:9200

  开启web界面:

 # cat /etc/graylog/server/server.conf|grep web_enable
 web_enable = true

 

  ④ 创建负载均衡器,对graylog配置负载均衡,我使用的是微软云负载均衡,这里不再说明。

  此时可以通过 负载均衡器IP:9000 对graylog进行访问。

 

5. 日志接入

接入 syslog

首先在 webui 创建 input:

CentOS 搭建Graylog集群详解

以 rsyslog 为例:

/etc/rsyslog.d/graylog.conf:

*.* @@x.x.x.x:514;RSYSLOG_SyslogProtocol23Format

service rsyslog restart

 

即可查看该 input 的 message:

CentOS 搭建Graylog集群详解

GELF (http 为例)

GELF (Graylog Extended Log Format) 可以接收结构化的事件, 支持压缩(GZIP’d or ZLIB’d)和分块。

GELF message:

    • version string
    • host string
    • short_message string
    • full_message string
    • timestamp number
    • level number
    • facility string
    • line number
    • file string
    • _[additional field] string or number, 通过 _ 前缀添加自定义的字段

新建一个 GELF HTTP input:

CentOS 搭建Graylog集群详解

推送日志:

curl -XPOST http://106.75.62.142:12201/gelf -p0 -d '{"message":"这是一条消息", "host":"172.3.3.3", "facility":"test", "topic": "meme"}'

 

查看推送的日志:

CentOS 搭建Graylog集群详解

收集服务日志( nodejs 为例)

log4js, bunyan, winston 等等 nodejs 日志框架都可以, 这里我们以 bunyan 为例, 因为 bunyan 可以将日志以 json 的形式打印。

const express = require('express');
const bodyParser = require('body-parser');
const bunyan = require('bunyan');

const log = bunyan.createLogger({
    name: 'server-bunyan',
    level: 'debug',
    streams: [{
        type: 'rotating-file',
        path: '/data/logs/server-bunyan.log',
        period: '1d',
        count: 3
    }]
});


const app = express();
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));

app.get('/hello', (req, res) => {
    log.info({                                           
        query: req.query
    }, 'hello');
    res.send('hello world');
});

app.listen(5004, '0.0.0.0', () => {
    log.info('app listening on 5004');
});

rsyslog:

module(load="imfile" PollingInterval="10")

# input
input(type
="imfile" File="/data/logs/server.log" Tag="server" ruleset="push_remote") input(type="imfile" File="/data/logs/detail.log" Tag="detail" ruleset="push_remote") input(type="imfile" File="/data/logs/server-bunyan.log" Tag="bunyan_server" ruleset="push_remote") # template template(name="mytpl" type="string" string="node1 %programname% %msg%\n" ) # output ruleset(name="push_remote") { action( type="omfwd" protocol="tcp" target="x.x.x.x" port="515" template="mytpl" action.resumeRetryCount="-1" action.resumeInterval="1" queue.filename="push-remote" queue.size="100000" queue.highwatermark="60000" queue.lowwatermark="2000" queue.maxdiskspace="100g" queue.saveonshutdown="on" queue.type="LinkedList" queue.maxfilesize="128m" ) } 

 

新建 input, 监听 515 端口,这里我们体验一下 graylog 的 Extractor,给改 input 添加一个 Extractor:

CentOS 搭建Graylog集群详解

我们加了一个抓取器,来提取 nodetopic 两个字段。

在 webui 查看该 input 的 message:

CentOS 搭建Graylog集群详解

Alerts

Graylog 内置的告警条件:

    • 消息数量
    • 字段值(number)
    • 字段内容

内置告警方式:

    • Email
    • HTTP 回调

体验一下 HTTP 回调。

新建一个 Stream, 进入 manager alerts, 新建一个告警条件:

CentOS 搭建Graylog集群详解

创建一个 HTTP 回调:

CentOS 搭建Graylog集群详解

告警以 post 方式请求回调, 请求的 body 内容:

{
    "check_result": {
        "result_description": "Stream had 0 messages in the last 1 minutes with trigger condition less than 10 messages. (Current grace time: 1 minutes)",
        "triggered_condition": {
            "id": "6bacc1c1-1eac-49f9-9ac8-998ea851f101",
            "type": "message_count",
            "created_at": "2017-01-17T05:25:13.592Z",
            "creator_user_id": "admin",
            "title": "日志一分钟内少于10条",
            "parameters": {
                "grace": 1,
                "threshold_type": "less",
                "threshold": 10,
                "time": 1,
                "backlog": 0
            }
        },
        "triggered_at": "2017-01-17T05:44:11.921Z",
        "triggered": true,
        "matching_messages": []
    },
    "stream": {
        "creator_user_id": "admin",
        "outputs": [],
        "alert_receivers": {
            "emails": [
                "dongsoso@hotmail.com"
            ],
            "users": [
                "dongsoso@hotmail.com"
            ]
        },
        "matching_type": "AND",
        "description": "alert",
        "created_at": "2017-01-17T05:21:58.852Z",
        "disabled": false,
        "rules": [],
        "alert_conditions": [
            {
                "creator_user_id": "admin",
                "created_at": "2017-01-17T05:25:13.592Z",
                "id": "6bacc1c1-1eac-49f9-9ac8-998ea851f101",
                "type": "message_count",
                "title": "日志一分钟内少于10条",
                "parameters": {
                    "grace": 1,
                    "threshold_type": "less",
                    "threshold": 10,
                    "time": 1,
                    "backlog": 0
                }
            }
        ],
        "id": "587da9f62ab79c0001352b7a",
        "title": "test",
        "content_pack": null
    }
} 

查看告警历史:

CentOS 搭建Graylog集群详解

更多更好用的功能等待发现…

官方文档 : http://docs.graylog.org/en/2.3/index.html

0

发表评论