福福不服 福福不服
深圳 --°
正在加载音乐 请稍候

ELK钉钉机器人Watch多条件告警

 ·  Elastic

告警效果

Watch

上星期发的那篇条件有点单一

告警会发现所有环境的系统日志告警全部混在一起的

这次还需要同时判断两个字段的信息，具体看Watch JSON

现在我这里有4条正在运行的高级watcher

develop_warn： develop环境10分钟内出现10次warn便钉钉告警列出10次告警内容

error_watcher： develop、production、sandbox三个环境只要一分钟内出现error就告警并列出告警内容

production_warn： production环境10分钟内出现10次warn便钉钉告警列出10次告警内容

sandbox_warn： sandbox环境10分钟内出现10次warn便钉钉告警列出10次告警内容

Watch JSON

develop_warn的Watch JSON

{
  "trigger": {
    "schedule": {
      "interval": "10m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "easyspeed-cloud-logs-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 10,
          "query": {
            "bool": {
              "must": [
                {
                  "match": {
                    "level": "WARN"
                  }
                },
                {
                  "match": {
                    "type": "develop"
                  }
                },
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-10m",
                      "lte": "now"
                    }
                  }
                }
              ]
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gte": 10
      }
    }
  },
  "actions": {
    "82da16ed-0763-4fb8-a19d-e95e29a418d3": {
      "webhook": {
        "scheme": "https",
        "host": "oapi.dingtalk.com",
        "port": 443,
        "method": "post",
        "path": "/robot/send",
        "params": {
          "access_token": "1a116f8bdd053xxxxxxxxxxxxxxxxxxxx"
        },
        "headers": {},
        "body": """{"msgtype":"text","text":{"content":"develop环境过去10分钟发现10次WARN，错误信息如下：
------------------------第 1次WARN信息------------------------
_index：{{ctx.payload.hits.hits.0._index}},
source：{{ctx.payload.hits.hits.0._source.source}} ,
traceId：{{ctx.payload.hits.hits.0._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.0._source.spanId}},
type：{{ctx.payload.hits.hits.0._source.type}} ,
host：{{ctx.payload.hits.hits.0._source.host}}
------------------------第 2次WARN信息------------------------
_index：{{ctx.payload.hits.hits.1._index}}，
source：{{ctx.payload.hits.hits.1._source.source}}, 
traceId：{{ctx.payload.hits.hits.1._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.1._source.spanId}}, 
type：{{ctx.payload.hits.hits.1._source.type}} ,
host：{{ctx.payload.hits.hits.1._source.host}}
------------------------第 3次WARN信息------------------------
_index：{{ctx.payload.hits.hits.2._index}}，
source：{{ctx.payload.hits.hits.2._source.source}}, 
traceId：{{ctx.payload.hits.hits.2._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.2._source.spanId}}, 
type：{{ctx.payload.hits.hits.2._source.type}} ,
host：{{ctx.payload.hits.hits.2._source.host}}
------------------------第 4次WARN信息------------------------
_index：{{ctx.payload.hits.hits.3._index}}，
source：{{ctx.payload.hits.hits.3._source.source}}, 
traceId：{{ctx.payload.hits.hits.3._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.3._source.spanId}}, 
type：{{ctx.payload.hits.hits.3._source.type}} ,
host：{{ctx.payload.hits.hits.3._source.host}}
------------------------第 5次WARN信息------------------------
_index：{{ctx.payload.hits.hits.4._index}}，
source：{{ctx.payload.hits.hits.4._source.source}}, 
traceId：{{ctx.payload.hits.hits.4._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.4._source.spanId}}, 
type：{{ctx.payload.hits.hits.4._source.type}} ,
host：{{ctx.payload.hits.hits.4._source.host}}
------------------------第6次WARN信息------------------------
_index：{{ctx.payload.hits.hits.5._index}}，
source：{{ctx.payload.hits.hits.5._source.source}}, 
traceId：{{ctx.payload.hits.hits.5._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.5._source.spanId}}, 
type：{{ctx.payload.hits.hits.5._source.type}} ,
host：{{ctx.payload.hits.hits.5._source.host}}
------------------------第 7次WARN信息------------------------
_index：{{ctx.payload.hits.hits.6._index}}，
source：{{ctx.payload.hits.hits.6._source.source}}, 
traceId：{{ctx.payload.hits.hits.6._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.6._source.spanId}}, 
type：{{ctx.payload.hits.hits.6._source.type}} ,
host：{{ctx.payload.hits.hits.6._source.host}}
------------------------第 8次WARN信息------------------------
_index：{{ctx.payload.hits.hits.7._index}}，
source：{{ctx.payload.hits.hits.7._source.source}}, 
traceId：{{ctx.payload.hits.hits.7._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.7._source.spanId}}, 
type：{{ctx.payload.hits.hits.7._source.type}} ,
host：{{ctx.payload.hits.hits.7._source.host}}
------------------------第 9次WARN信息------------------------
_index：{{ctx.payload.hits.hits.8._index}}，
source：{{ctx.payload.hits.hits.8._source.source}}, 
traceId：{{ctx.payload.hits.hits.8._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.8._source.spanId}}, 
type：{{ctx.payload.hits.hits.8._source.type}} ,
host：{{ctx.payload.hits.hits.8._source.host}}
------------------------第 10次WARN信息------------------------
_index：{{ctx.payload.hits.hits.9._index}}，
source：{{ctx.payload.hits.hits.9._source.source}}, 
traceId：{{ctx.payload.hits.hits.9._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.9._source.spanId}}, 
type：{{ctx.payload.hits.hits.9._source.type}} ,
host：{{ctx.payload.hits.hits.9._source.host}}
------------------------      END      ------------------------
详情信息请登录查看：http://logs.hkeasyspeed.com/ "}}"""
      }
    }
  }
}

error_watcher的Watch JSON

{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "easyspeed-cloud-logs-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 1,
          "query": {
            "bool": {
              "must": [
                {
                  "match": {
                    "level": "ERROR"
                  }
                },
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-1m",
                      "lte": "now"
                    }
                  }
                }
              ]
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gte": 1
      }
    }
  },
  "actions": {
    "a909971b-2065-46a4-8e71-5208f8c8abd0": {
      "webhook": {
        "scheme": "https",
        "host": "oapi.dingtalk.com",
        "port": 443,
        "method": "post",
        "path": "/robot/send",
        "params": {
          "access_token": "1a116f8bdd053xxxxxxxxxxxxxxxxxxxx"
        },
        "headers": {},
        "body": """{"msgtype":"text","text":{"content":"发现ERROR，错误信息如下：
_index：{{ctx.payload.hits.hits.0._index}}，
source：{{ctx.payload.hits.hits.0._source.source}}, 
traceId：{{ctx.payload.hits.hits.0._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.0._source.spanId}}, 
type：{{ctx.payload.hits.hits.0._source.type}},
host：{{ctx.payload.hits.hits.0._source.host}}
详情信息请登录查看：http://logs.hkeasyspeed.com/ "}}"""
      }
    }
  }
}

production_warn的Watch JSON

{
  "trigger": {
    "schedule": {
      "interval": "10m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "easyspeed-cloud-logs-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 10,
          "query": {
            "bool": {
              "must": [
                {
                  "match": {
                    "level": "WARN"
                  }
                },
                {
                  "match": {
                    "type": "production"
                  }
                },
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-10m",
                      "lte": "now"
                    }
                  }
                }
              ]
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gte": 10
      }
    }
  },
  "actions": {
    "2a170180-1301-4e19-aae1-ec2f0d7deae1": {
      "webhook": {
        "scheme": "https",
        "host": "oapi.dingtalk.com",
        "port": 443,
        "method": "post",
        "path": "/robot/send",
        "params": {
          "access_token": "1a116f8bdd053xxxxxxxxxxxxxxxxxxxx"
        },
        "headers": {},
        "body": """{"msgtype":"text","text":{"content":"production环境过去10分钟发现10次WARN，错误信息如下：
------------------------第 1次WARN信息------------------------
_index：{{ctx.payload.hits.hits.0._index}}，
source：{{ctx.payload.hits.hits.0._source.source}}, 
traceId：{{ctx.payload.hits.hits.0._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.0._source.spanId}}, 
type：{{ctx.payload.hits.hits.0._source.type}} ,
host：{{ctx.payload.hits.hits.0._source.host}}
------------------------第 2次WARN信息------------------------
_index：{{ctx.payload.hits.hits.1._index}}，
source：{{ctx.payload.hits.hits.1._source.source}}, 
traceId：{{ctx.payload.hits.hits.1._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.1._source.spanId}}, 
type：{{ctx.payload.hits.hits.1._source.type}} ,
host：{{ctx.payload.hits.hits.1._source.host}}
------------------------第 3次WARN信息------------------------
_index：{{ctx.payload.hits.hits.2._index}}，
source：{{ctx.payload.hits.hits.2._source.source}}, 
traceId：{{ctx.payload.hits.hits.2._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.2._source.spanId}}, 
type：{{ctx.payload.hits.hits.2._source.type}} ,
host：{{ctx.payload.hits.hits.2._source.host}}
------------------------第 4次WARN信息------------------------
_index：{{ctx.payload.hits.hits.3._index}}，
source：{{ctx.payload.hits.hits.3._source.source}}, 
traceId：{{ctx.payload.hits.hits.3._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.3._source.spanId}}, 
type：{{ctx.payload.hits.hits.3._source.type}} ,
host：{{ctx.payload.hits.hits.3._source.host}}
------------------------第 5次WARN信息------------------------
_index：{{ctx.payload.hits.hits.4._index}}，
source：{{ctx.payload.hits.hits.4._source.source}}, 
traceId：{{ctx.payload.hits.hits.4._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.4._source.spanId}}, 
type：{{ctx.payload.hits.hits.4._source.type}} ,
host：{{ctx.payload.hits.hits.4._source.host}}
------------------------第6次WARN信息------------------------
_index：{{ctx.payload.hits.hits.5._index}}，
source：{{ctx.payload.hits.hits.5._source.source}}, 
traceId：{{ctx.payload.hits.hits.5._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.5._source.spanId}}, 
type：{{ctx.payload.hits.hits.5._source.type}} ,
host：{{ctx.payload.hits.hits.5._source.host}}
------------------------第 7次WARN信息------------------------
_index：{{ctx.payload.hits.hits.6._index}}，
source：{{ctx.payload.hits.hits.6._source.source}}, 
traceId：{{ctx.payload.hits.hits.6._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.6._source.spanId}}, 
type：{{ctx.payload.hits.hits.6._source.type}} ,
host：{{ctx.payload.hits.hits.6._source.host}}
------------------------第 8次WARN信息------------------------
_index：{{ctx.payload.hits.hits.7._index}}，
source：{{ctx.payload.hits.hits.7._source.source}}, 
traceId：{{ctx.payload.hits.hits.7._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.7._source.spanId}}, 
type：{{ctx.payload.hits.hits.7._source.type}} ,
host：{{ctx.payload.hits.hits.7._source.host}}
------------------------第 9次WARN信息------------------------
_index：{{ctx.payload.hits.hits.8._index}}，
source：{{ctx.payload.hits.hits.8._source.source}}, 
traceId：{{ctx.payload.hits.hits.8._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.8._source.spanId}}, 
type：{{ctx.payload.hits.hits.8._source.type}} ,
host：{{ctx.payload.hits.hits.8._source.host}}
------------------------第 10次WARN信息------------------------
_index：{{ctx.payload.hits.hits.9._index}}，
source：{{ctx.payload.hits.hits.9._source.source}}, 
traceId：{{ctx.payload.hits.hits.9._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.9._source.spanId}}, 
type：{{ctx.payload.hits.hits.9._source.type}} ,
host：{{ctx.payload.hits.hits.9._source.host}}
------------------------      END      ------------------------
详情信息请登录查看：http://logs.hkeasyspeed.com/ "}}"""
      }
    }
  }
}

sandbox_warn的Watch JSON

{
  "trigger": {
    "schedule": {
      "interval": "10m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "easyspeed-cloud-logs-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 10,
          "query": {
            "bool": {
              "must": [
                {
                  "match": {
                    "level": "WARN"
                  }
                },
                {
                  "match": {
                    "type": "sandbox"
                  }
                },
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-10m",
                      "lte": "now"
                    }
                  }
                }
              ]
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gte": 10
      }
    }
  },
  "actions": {
    "ec87be74-c6e6-4776-a238-53a8e0091f9c": {
      "webhook": {
        "scheme": "https",
        "host": "oapi.dingtalk.com",
        "port": 443,
        "method": "post",
        "path": "/robot/send",
        "params": {
          "access_token": "1a116f8bdd053xxxxxxxxxxxxxxxxxxxx"
        },
        "headers": {},
        "body": """{"msgtype":"text","text":{"content":"沙盒环境过去10分钟发现10次WARN，错误信息如下：
------------------------第 1次WARN信息------------------------
_index：{{ctx.payload.hits.hits.0._index}},
source：{{ctx.payload.hits.hits.0._source.source}} ,
traceId：{{ctx.payload.hits.hits.0._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.0._source.spanId}},
type：{{ctx.payload.hits.hits.0._source.type}} ,
host：{{ctx.payload.hits.hits.0._source.host}}
------------------------第 2次WARN信息------------------------
_index：{{ctx.payload.hits.hits.1._index}}，
source：{{ctx.payload.hits.hits.1._source.source}}, 
traceId：{{ctx.payload.hits.hits.1._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.1._source.spanId}}, 
type：{{ctx.payload.hits.hits.1._source.type}} ,
host：{{ctx.payload.hits.hits.1._source.host}}
------------------------第 3次WARN信息------------------------
_index：{{ctx.payload.hits.hits.2._index}}，
source：{{ctx.payload.hits.hits.2._source.source}}, 
traceId：{{ctx.payload.hits.hits.2._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.2._source.spanId}}, 
type：{{ctx.payload.hits.hits.2._source.type}} ,
host：{{ctx.payload.hits.hits.2._source.host}}
------------------------第 4次WARN信息------------------------
_index：{{ctx.payload.hits.hits.3._index}}，
source：{{ctx.payload.hits.hits.3._source.source}}, 
traceId：{{ctx.payload.hits.hits.3._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.3._source.spanId}}, 
type：{{ctx.payload.hits.hits.3._source.type}} ,
host：{{ctx.payload.hits.hits.3._source.host}}
------------------------第 5次WARN信息------------------------
_index：{{ctx.payload.hits.hits.4._index}}，
source：{{ctx.payload.hits.hits.4._source.source}}, 
traceId：{{ctx.payload.hits.hits.4._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.4._source.spanId}}, 
type：{{ctx.payload.hits.hits.4._source.type}} ,
host：{{ctx.payload.hits.hits.4._source.host}}
------------------------第6次WARN信息------------------------
_index：{{ctx.payload.hits.hits.5._index}}，
source：{{ctx.payload.hits.hits.5._source.source}}, 
traceId：{{ctx.payload.hits.hits.5._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.5._source.spanId}}, 
type：{{ctx.payload.hits.hits.5._source.type}} ,
host：{{ctx.payload.hits.hits.5._source.host}}
------------------------第 7次WARN信息------------------------
_index：{{ctx.payload.hits.hits.6._index}}，
source：{{ctx.payload.hits.hits.6._source.source}}, 
traceId：{{ctx.payload.hits.hits.6._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.6._source.spanId}}, 
type：{{ctx.payload.hits.hits.6._source.type}} ,
host：{{ctx.payload.hits.hits.6._source.host}}
------------------------第 8次WARN信息------------------------
_index：{{ctx.payload.hits.hits.7._index}}，
source：{{ctx.payload.hits.hits.7._source.source}}, 
traceId：{{ctx.payload.hits.hits.7._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.7._source.spanId}}, 
type：{{ctx.payload.hits.hits.7._source.type}} ,
host：{{ctx.payload.hits.hits.7._source.host}}
------------------------第 9次WARN信息------------------------
_index：{{ctx.payload.hits.hits.8._index}}，
source：{{ctx.payload.hits.hits.8._source.source}}, 
traceId：{{ctx.payload.hits.hits.8._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.8._source.spanId}}, 
type：{{ctx.payload.hits.hits.8._source.type}} ,
host：{{ctx.payload.hits.hits.8._source.host}}
------------------------第 10次WARN信息------------------------
_index：{{ctx.payload.hits.hits.9._index}}，
source：{{ctx.payload.hits.hits.9._source.source}}, 
traceId：{{ctx.payload.hits.hits.9._source.traceId}} ,
spanId：{{ctx.payload.hits.hits.9._source.spanId}}, 
type：{{ctx.payload.hits.hits.9._source.type}} ,
host：{{ctx.payload.hits.hits.9._source.host}}
------------------------      END      ------------------------
详情信息请登录查看：http://logs.hkeasyspeed.com/ "}}"""
      }
    }
  }
}

KibanaLogstashElasticsearch