Merge pull request #291 from AkkiaS7/fix-service-recover-notification
服务状态变更时清除静音缓存 Co-authored-by: AkkiaS7 <68485070+AkkiaS7@users.noreply.github.com>
This commit is contained in:
commit
2edfff334f
@ -118,12 +118,16 @@ func (s *NezhaHandler) ReportSystemInfo(c context.Context, r *pb.Host) (*pb.Rece
|
|||||||
host.IP != "" &&
|
host.IP != "" &&
|
||||||
singleton.ServerList[clientID].Host.IP != host.IP {
|
singleton.ServerList[clientID].Host.IP != host.IP {
|
||||||
|
|
||||||
singleton.SendNotification(singleton.Conf.IPChangeNotificationTag, fmt.Sprintf(
|
singleton.SendNotification(singleton.Conf.IPChangeNotificationTag,
|
||||||
"[%s] %s, %s => %s",
|
fmt.Sprintf(
|
||||||
singleton.Localizer.MustLocalize(&i18n.LocalizeConfig{
|
"[%s] %s, %s => %s",
|
||||||
MessageID: "IPChanged",
|
singleton.Localizer.MustLocalize(&i18n.LocalizeConfig{
|
||||||
}),
|
MessageID: "IPChanged",
|
||||||
singleton.ServerList[clientID].Name, singleton.IPDesensitize(singleton.ServerList[clientID].Host.IP), singleton.IPDesensitize(host.IP)), singleton.NotificationMuteLabel.IPChanged(clientID), nil)
|
}),
|
||||||
|
singleton.ServerList[clientID].Name, singleton.IPDesensitize(singleton.ServerList[clientID].Host.IP),
|
||||||
|
singleton.IPDesensitize(host.IP),
|
||||||
|
),
|
||||||
|
nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 判断是否是机器重启,如果是机器重启要录入最后记录的流量里面
|
// 判断是否是机器重启,如果是机器重启要录入最后记录的流量里面
|
||||||
|
@ -165,8 +165,7 @@ func checkStatus() {
|
|||||||
go SendTriggerTasks(alert.FailTriggerTasks, curServer.ID)
|
go SendTriggerTasks(alert.FailTriggerTasks, curServer.ID)
|
||||||
go SendNotification(alert.NotificationTag, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), &curServer)
|
go SendNotification(alert.NotificationTag, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), &curServer)
|
||||||
// 清除恢复通知的静音缓存
|
// 清除恢复通知的静音缓存
|
||||||
resolvedMuteLabel := NotificationMuteLabel.AppendNotificationTag(NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), alert.NotificationTag)
|
UnMuteNotification(alert.NotificationTag, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID))
|
||||||
Cache.Delete(*resolvedMuteLabel)
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// 本次通过检查但上一次的状态为失败,则发送恢复通知
|
// 本次通过检查但上一次的状态为失败,则发送恢复通知
|
||||||
@ -177,8 +176,7 @@ func checkStatus() {
|
|||||||
go SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID)
|
go SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID)
|
||||||
go SendNotification(alert.NotificationTag, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), &curServer)
|
go SendNotification(alert.NotificationTag, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), &curServer)
|
||||||
// 清除失败通知的静音缓存
|
// 清除失败通知的静音缓存
|
||||||
incidentMuteLabel := NotificationMuteLabel.AppendNotificationTag(NotificationMuteLabel.ServerIncident(server.ID, alert.ID), alert.NotificationTag)
|
UnMuteNotification(alert.NotificationTag, NotificationMuteLabel.ServerIncident(server.ID, alert.ID))
|
||||||
Cache.Delete(*incidentMuteLabel)
|
|
||||||
}
|
}
|
||||||
alertsPrevState[alert.ID][server.ID] = _RuleCheckPass
|
alertsPrevState[alert.ID][server.ID] = _RuleCheckPass
|
||||||
}
|
}
|
||||||
|
@ -102,6 +102,11 @@ func OnDeleteNotification(id uint64) {
|
|||||||
delete(NotificationIDToTag, id)
|
delete(NotificationIDToTag, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func UnMuteNotification(notificationTag string, muteLabel *string) {
|
||||||
|
fullMuteLabel := *NotificationMuteLabel.AppendNotificationTag(muteLabel, notificationTag)
|
||||||
|
Cache.Delete(fullMuteLabel)
|
||||||
|
}
|
||||||
|
|
||||||
// SendNotification 向指定的通知方式组的所有通知方式发送通知
|
// SendNotification 向指定的通知方式组的所有通知方式发送通知
|
||||||
func SendNotification(notificationTag string, desc string, muteLabel *string, ext ...*model.Server) {
|
func SendNotification(notificationTag string, desc string, muteLabel *string, ext ...*model.Server) {
|
||||||
if muteLabel != nil {
|
if muteLabel != nil {
|
||||||
@ -199,7 +204,7 @@ func (_NotificationMuteLabel) ServiceStateChanged(serviceId uint64) *string {
|
|||||||
return &label
|
return &label
|
||||||
}
|
}
|
||||||
|
|
||||||
func (_NotificationMuteLabel) ServiceSSL(serviceId uint64) *string {
|
func (_NotificationMuteLabel) ServiceSSL(serviceId uint64, extraInfo string) *string {
|
||||||
label := fmt.Sprintf("bf::sssl-%d", serviceId)
|
label := fmt.Sprintf("bf::sssl-%d-%s", serviceId, extraInfo)
|
||||||
return &label
|
return &label
|
||||||
}
|
}
|
||||||
|
@ -382,103 +382,151 @@ func (ss *ServiceSentinel) worker() {
|
|||||||
if mh.Delay > 0 {
|
if mh.Delay > 0 {
|
||||||
ss.monitorsLock.RLock()
|
ss.monitorsLock.RLock()
|
||||||
if ss.monitors[mh.GetId()].LatencyNotify {
|
if ss.monitors[mh.GetId()].LatencyNotify {
|
||||||
|
notificationTag := ss.monitors[mh.GetId()].NotificationTag
|
||||||
|
minMuteLabel := NotificationMuteLabel.ServiceLatencyMin(mh.GetId())
|
||||||
|
maxMuteLabel := NotificationMuteLabel.ServiceLatencyMax(mh.GetId())
|
||||||
if mh.Delay > ss.monitors[mh.GetId()].MaxLatency {
|
if mh.Delay > ss.monitors[mh.GetId()].MaxLatency {
|
||||||
|
// 延迟超过最大值
|
||||||
ServerLock.RLock()
|
ServerLock.RLock()
|
||||||
reporterServer := ServerList[r.Reporter]
|
reporterServer := ServerList[r.Reporter]
|
||||||
msg := fmt.Sprintf("[Latency] %s %2f > %2f, Reporter: %s", ss.monitors[mh.GetId()].Name, mh.Delay, ss.monitors[mh.GetId()].MaxLatency, reporterServer.Name)
|
msg := fmt.Sprintf("[Latency] %s %2f > %2f, Reporter: %s", ss.monitors[mh.GetId()].Name, mh.Delay, ss.monitors[mh.GetId()].MaxLatency, reporterServer.Name)
|
||||||
muteLabel := NotificationMuteLabel.ServiceLatencyMin(mh.GetId())
|
go SendNotification(notificationTag, msg, minMuteLabel)
|
||||||
go SendNotification(ss.monitors[mh.GetId()].NotificationTag, msg, muteLabel)
|
|
||||||
ServerLock.RUnlock()
|
ServerLock.RUnlock()
|
||||||
}
|
} else if mh.Delay < ss.monitors[mh.GetId()].MinLatency {
|
||||||
if mh.Delay < ss.monitors[mh.GetId()].MinLatency {
|
// 延迟低于最小值
|
||||||
ServerLock.RLock()
|
ServerLock.RLock()
|
||||||
reporterServer := ServerList[r.Reporter]
|
reporterServer := ServerList[r.Reporter]
|
||||||
go SendNotification(ss.monitors[mh.GetId()].NotificationTag, fmt.Sprintf("[Latency] %s %2f < %2f, Reporter: %s", ss.monitors[mh.GetId()].Name, mh.Delay, ss.monitors[mh.GetId()].MinLatency, reporterServer.Name), NotificationMuteLabel.ServiceLatencyMax(mh.GetId()))
|
msg := fmt.Sprintf("[Latency] %s %2f < %2f, Reporter: %s", ss.monitors[mh.GetId()].Name, mh.Delay, ss.monitors[mh.GetId()].MinLatency, reporterServer.Name)
|
||||||
|
go SendNotification(notificationTag, msg, maxMuteLabel)
|
||||||
ServerLock.RUnlock()
|
ServerLock.RUnlock()
|
||||||
|
} else {
|
||||||
|
// 正常延迟, 清除静音缓存
|
||||||
|
UnMuteNotification(notificationTag, minMuteLabel)
|
||||||
|
UnMuteNotification(notificationTag, maxMuteLabel)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ss.monitorsLock.RUnlock()
|
ss.monitorsLock.RUnlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
// 状态变更报警
|
// 状态变更报警+触发任务执行
|
||||||
if stateCode == StatusDown || stateCode != ss.lastStatus[mh.GetId()] {
|
if stateCode == StatusDown || stateCode != ss.lastStatus[mh.GetId()] {
|
||||||
ss.monitorsLock.Lock()
|
ss.monitorsLock.Lock()
|
||||||
lastStatus := ss.lastStatus[mh.GetId()]
|
lastStatus := ss.lastStatus[mh.GetId()]
|
||||||
// 存储新的状态值
|
// 存储新的状态值
|
||||||
ss.lastStatus[mh.GetId()] = stateCode
|
ss.lastStatus[mh.GetId()] = stateCode
|
||||||
|
|
||||||
// 判断是否需要发送提醒
|
// 判断是否需要发送通知
|
||||||
isNeedSendNotification := ss.monitors[mh.GetId()].Notify && (lastStatus != 0 || stateCode == StatusDown)
|
isNeedSendNotification := ss.monitors[mh.GetId()].Notify && (lastStatus != 0 || stateCode == StatusDown)
|
||||||
if isNeedSendNotification {
|
if isNeedSendNotification {
|
||||||
ServerLock.RLock()
|
ServerLock.RLock()
|
||||||
reporterServer := ServerList[r.Reporter]
|
|
||||||
go SendNotification(ss.monitors[mh.GetId()].NotificationTag, fmt.Sprintf("[%s] %s Reporter: %s, Error: %s", StatusCodeToString(stateCode), ss.monitors[mh.GetId()].Name, reporterServer.Name, mh.Data), NotificationMuteLabel.ServiceStateChanged(mh.GetId()))
|
|
||||||
|
|
||||||
|
reporterServer := ServerList[r.Reporter]
|
||||||
|
notificationTag := ss.monitors[mh.GetId()].NotificationTag
|
||||||
|
notificationMsg := fmt.Sprintf("[%s] %s Reporter: %s, Error: %s", StatusCodeToString(stateCode), ss.monitors[mh.GetId()].Name, reporterServer.Name, mh.Data)
|
||||||
|
muteLabel := NotificationMuteLabel.ServiceStateChanged(mh.GetId())
|
||||||
|
|
||||||
|
// 状态变更时,清除静音缓存
|
||||||
|
if stateCode != lastStatus {
|
||||||
|
UnMuteNotification(notificationTag, muteLabel)
|
||||||
|
}
|
||||||
|
|
||||||
|
go SendNotification(notificationTag, notificationMsg, muteLabel)
|
||||||
ServerLock.RUnlock()
|
ServerLock.RUnlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
// 判断是否需要触发任务
|
// 判断是否需要触发任务
|
||||||
if ss.monitors[mh.GetId()].EnableTriggerTask && lastStatus != 0 {
|
isNeedTriggerTask := ss.monitors[mh.GetId()].EnableTriggerTask && lastStatus != 0
|
||||||
|
if isNeedTriggerTask {
|
||||||
ServerLock.RLock()
|
ServerLock.RLock()
|
||||||
reporterServer := ServerList[r.Reporter]
|
reporterServer := ServerList[r.Reporter]
|
||||||
ServerLock.RUnlock()
|
ServerLock.RUnlock()
|
||||||
|
|
||||||
if stateCode == StatusGood && lastStatus != stateCode {
|
if stateCode == StatusGood && lastStatus != stateCode {
|
||||||
// 当前状态正常 前序状态异常时 触发恢复任务
|
// 当前状态正常 前序状态非正常时 触发恢复任务
|
||||||
go SendTriggerTasks(ss.monitors[mh.GetId()].RecoverTriggerTasks, reporterServer.ID)
|
go SendTriggerTasks(ss.monitors[mh.GetId()].RecoverTriggerTasks, reporterServer.ID)
|
||||||
} else if lastStatus == StatusGood && lastStatus != stateCode {
|
} else if lastStatus == StatusGood && lastStatus != stateCode {
|
||||||
// 前序状态正常 当前状态异常时 触发失败任务
|
// 前序状态正常 当前状态非正常时 触发失败任务
|
||||||
go SendTriggerTasks(ss.monitors[mh.GetId()].FailTriggerTasks, reporterServer.ID)
|
go SendTriggerTasks(ss.monitors[mh.GetId()].FailTriggerTasks, reporterServer.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// 当前状态正常 前序状态非正常时触发恢复任务
|
|
||||||
|
|
||||||
ss.monitorsLock.Unlock()
|
ss.monitorsLock.Unlock()
|
||||||
}
|
}
|
||||||
ss.serviceResponseDataStoreLock.Unlock()
|
ss.serviceResponseDataStoreLock.Unlock()
|
||||||
|
|
||||||
// SSL 证书报警
|
// SSL 证书报警
|
||||||
var errMsg string
|
var errMsg string
|
||||||
if strings.HasPrefix(mh.Data, "SSL证书错误:") {
|
if strings.HasPrefix(mh.Data, "SSL证书错误:") {
|
||||||
// 排除 i/o timeout、connection timeout、EOF 错误
|
// i/o timeout、connection timeout、EOF 错误
|
||||||
if !strings.HasSuffix(mh.Data, "timeout") &&
|
if !strings.HasSuffix(mh.Data, "timeout") &&
|
||||||
!strings.HasSuffix(mh.Data, "EOF") &&
|
!strings.HasSuffix(mh.Data, "EOF") &&
|
||||||
!strings.HasSuffix(mh.Data, "timed out") {
|
!strings.HasSuffix(mh.Data, "timed out") {
|
||||||
errMsg = mh.Data
|
errMsg = mh.Data
|
||||||
|
ss.monitorsLock.RLock()
|
||||||
|
if ss.monitors[mh.GetId()].Notify {
|
||||||
|
muteLabel := NotificationMuteLabel.ServiceSSL(mh.GetId(), "network")
|
||||||
|
go SendNotification(ss.monitors[mh.GetId()].NotificationTag, fmt.Sprintf("[SSL] Fetch cert info failed, %s %s", ss.monitors[mh.GetId()].Name, errMsg), muteLabel)
|
||||||
|
}
|
||||||
|
ss.monitorsLock.RUnlock()
|
||||||
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
// 清除网络错误静音缓存
|
||||||
|
UnMuteNotification(ss.monitors[mh.GetId()].NotificationTag, NotificationMuteLabel.ServiceSSL(mh.GetId(), "network"))
|
||||||
|
|
||||||
var newCert = strings.Split(mh.Data, "|")
|
var newCert = strings.Split(mh.Data, "|")
|
||||||
if len(newCert) > 1 {
|
if len(newCert) > 1 {
|
||||||
|
ss.monitorsLock.Lock()
|
||||||
|
enableNotify := ss.monitors[mh.GetId()].Notify
|
||||||
|
|
||||||
|
// 首次获取证书信息时,缓存证书信息
|
||||||
if ss.sslCertCache[mh.GetId()] == "" {
|
if ss.sslCertCache[mh.GetId()] == "" {
|
||||||
ss.sslCertCache[mh.GetId()] = mh.Data
|
ss.sslCertCache[mh.GetId()] = mh.Data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
oldCert := strings.Split(ss.sslCertCache[mh.GetId()], "|")
|
||||||
|
isCertChanged := false
|
||||||
|
expiresOld, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", oldCert[1])
|
||||||
expiresNew, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", newCert[1])
|
expiresNew, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", newCert[1])
|
||||||
// 证书过期提醒
|
|
||||||
if expiresNew.Before(time.Now().AddDate(0, 0, 7)) {
|
// 证书变更时,更新缓存
|
||||||
errMsg = fmt.Sprintf(
|
|
||||||
"The SSL certificate will expire within seven days. Expiration time: %s",
|
|
||||||
expiresNew.Format("2006-01-02 15:04:05"))
|
|
||||||
}
|
|
||||||
// 证书变更提醒
|
|
||||||
var oldCert = strings.Split(ss.sslCertCache[mh.GetId()], "|")
|
|
||||||
var expiresOld time.Time
|
|
||||||
if len(oldCert) > 1 {
|
|
||||||
expiresOld, _ = time.Parse("2006-01-02 15:04:05 -0700 MST", oldCert[1])
|
|
||||||
}
|
|
||||||
if oldCert[0] != newCert[0] && !expiresNew.Equal(expiresOld) {
|
if oldCert[0] != newCert[0] && !expiresNew.Equal(expiresOld) {
|
||||||
|
isCertChanged = true
|
||||||
ss.sslCertCache[mh.GetId()] = mh.Data
|
ss.sslCertCache[mh.GetId()] = mh.Data
|
||||||
errMsg = fmt.Sprintf(
|
}
|
||||||
"SSL certificate changed, old: %s, %s expired; new: %s, %s expired.",
|
|
||||||
oldCert[0], expiresOld.Format("2006-01-02 15:04:05"), newCert[0], expiresNew.Format("2006-01-02 15:04:05"))
|
notificationTag := ss.monitors[mh.GetId()].NotificationTag
|
||||||
|
serviceName := ss.monitors[mh.GetId()].Name
|
||||||
|
ss.monitorsLock.Unlock()
|
||||||
|
|
||||||
|
// 需要发送提醒
|
||||||
|
if enableNotify {
|
||||||
|
// 证书过期提醒
|
||||||
|
if expiresNew.Before(time.Now().AddDate(0, 0, 7)) {
|
||||||
|
expiresTimeStr := expiresNew.Format("2006-01-02 15:04:05")
|
||||||
|
errMsg = fmt.Sprintf(
|
||||||
|
"The SSL certificate will expire within seven days. Expiration time: %s",
|
||||||
|
expiresTimeStr,
|
||||||
|
)
|
||||||
|
|
||||||
|
// 静音规则: 服务id+证书过期时间
|
||||||
|
// 用于避免多个监测点对相同证书同时报警
|
||||||
|
muteLabel := NotificationMuteLabel.ServiceSSL(mh.GetId(), fmt.Sprintf("expire_%s", expiresTimeStr))
|
||||||
|
go SendNotification(notificationTag, fmt.Sprintf("[SSL] %s %s", serviceName, errMsg), muteLabel)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 证书变更提醒
|
||||||
|
if isCertChanged {
|
||||||
|
errMsg = fmt.Sprintf(
|
||||||
|
"SSL certificate changed, old: %s, %s expired; new: %s, %s expired.",
|
||||||
|
oldCert[0], expiresOld.Format("2006-01-02 15:04:05"), newCert[0], expiresNew.Format("2006-01-02 15:04:05"))
|
||||||
|
|
||||||
|
// 证书变更后会自动更新缓存,所以不需要静音
|
||||||
|
go SendNotification(notificationTag, fmt.Sprintf("[SSL] %s %s", serviceName, errMsg), nil)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if errMsg != "" {
|
|
||||||
ss.monitorsLock.RLock()
|
|
||||||
if ss.monitors[mh.GetId()].Notify {
|
|
||||||
go SendNotification(ss.monitors[mh.GetId()].NotificationTag, fmt.Sprintf("[SSL] %s %s", ss.monitors[mh.GetId()].Name, errMsg), NotificationMuteLabel.ServiceSSL(mh.GetId()))
|
|
||||||
}
|
|
||||||
ss.monitorsLock.RUnlock()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user