| 1 |
|
//---PACKAGE DECLARATION--- |
| 2 |
< |
package uk.ac.ukc.iscream.client.monitors; |
| 2 |
> |
package uk.org.iscream.client.monitors; |
| 3 |
|
|
| 4 |
|
//---IMPORTS--- |
| 5 |
|
import java.util.HashMap; |
| 6 |
|
import java.util.Iterator; |
| 7 |
< |
import uk.ac.ukc.iscream.client.*; |
| 8 |
< |
import uk.ac.ukc.iscream.core.*; |
| 9 |
< |
import uk.ac.ukc.iscream.util.*; |
| 10 |
< |
import uk.ac.ukc.iscream.componentmanager.*; |
| 7 |
> |
import uk.org.iscream.client.*; |
| 8 |
> |
import uk.org.iscream.core.*; |
| 9 |
> |
import uk.org.iscream.util.*; |
| 10 |
> |
import uk.org.iscream.componentmanager.*; |
| 11 |
|
|
| 12 |
|
/** |
| 13 |
|
* This Monitor watches heartbeats |
| 15 |
|
* @author $Author$ |
| 16 |
|
* @version $Id$ |
| 17 |
|
*/ |
| 18 |
< |
public class Heartbeat__Monitor extends Thread implements PluginMonitor { |
| 18 |
> |
public class Heartbeat__Monitor extends MonitorSkeleton { |
| 19 |
|
|
| 20 |
|
//---FINAL ATTRIBUTES--- |
| 21 |
|
|
| 26 |
|
|
| 27 |
|
public final String DESC = "Monitors Heartbeats."; |
| 28 |
|
|
| 29 |
+ |
public final int DEFAULT_CHECK_PERIOD = 60; |
| 30 |
+ |
|
| 31 |
|
//---STATIC METHODS--- |
| 32 |
|
|
| 33 |
|
//---CONSTRUCTORS--- |
| 34 |
|
|
| 35 |
< |
//---PUBLIC METHODS--- |
| 36 |
< |
|
| 37 |
< |
public void run() { |
| 36 |
< |
ConfigurationProxy cp = ConfigurationProxy.getInstance(); |
| 37 |
< |
while(true) { |
| 38 |
< |
// this cycle period could be done better, maybe ? |
| 39 |
< |
int checkPeriod = Integer.parseInt(cp.getProperty(_name, "Monitor.Heartbeat.checkPeriod")); |
| 40 |
< |
|
| 41 |
< |
// perform the checks (use HB hash, although they *should* be the same) |
| 42 |
< |
Iterator i = _hostsHB.keySet().iterator(); |
| 43 |
< |
while(i.hasNext()) { |
| 44 |
< |
// get host |
| 45 |
< |
String source = (String) i.next(); |
| 46 |
< |
// check it |
| 47 |
< |
analyseHB(source); |
| 48 |
< |
} |
| 49 |
< |
|
| 50 |
< |
// wait a while |
| 51 |
< |
try {Thread.sleep(checkPeriod * 1000);} catch (InterruptedException e) {} |
| 52 |
< |
} |
| 35 |
> |
public Heartbeat__Monitor() { |
| 36 |
> |
super(); |
| 37 |
> |
new HeartbeatWorker().start(); |
| 38 |
|
} |
| 54 |
– |
|
| 55 |
– |
// only use attribute num 0 :) |
| 56 |
– |
public void analyseHB(String source) { |
| 57 |
– |
ConfigurationProxy cp = ConfigurationProxy.getInstance(); |
| 58 |
– |
Register reg = (Register) _hostsReg.get(source); |
| 59 |
– |
|
| 60 |
– |
// get host's HB interval (seconds) |
| 61 |
– |
int hostHBinterval = Integer.parseInt(cp.getProperty("Host."+source, "TCPUpdateTime")); |
| 62 |
– |
// get host's last HB time (seconds) |
| 63 |
– |
int lastHeartbeat = ((HeartbeatHolder) _hostsHB.get(source)).getLastHeartbeat(); |
| 64 |
– |
// time since last heartbeat (seconds) |
| 65 |
– |
int timeSinceLastHB = ((int) (System.currentTimeMillis()/1000)) - lastHeartbeat; |
| 66 |
– |
|
| 67 |
– |
// find out the threshold level we're at |
| 68 |
– |
int result = checkAttributeThreshold(timeSinceLastHB, reg); |
| 69 |
– |
|
| 70 |
– |
// decide what threshold level we're on, if we've changed, record that |
| 71 |
– |
if (result != reg.getLastThresholdLevel(0)) { |
| 72 |
– |
reg.setLastThresholdLevel(0, result); |
| 73 |
– |
} |
| 74 |
– |
|
| 75 |
– |
// as long as this isn't a normal level |
| 76 |
– |
if(reg.getLastThresholdLevel(0) != Alert.thresholdNORMAL) { |
| 77 |
– |
// if the time since the last alert is more than the time for |
| 78 |
– |
// its timeout, fire an alert, escalate the alert |
| 79 |
– |
long timeout = reg.getLastAlertTimeout(0); |
| 80 |
– |
if ((timeout > 0) && (reg.getTimeLastSent(0) > 0)) { |
| 81 |
– |
if((System.currentTimeMillis() - reg.getTimeLastSent(0)) > timeout) { |
| 82 |
– |
int lastAlert = reg.getLastAlertLevel(0); |
| 83 |
– |
reg.escalateAlert(0); |
| 84 |
– |
reg.setTimeLastSent(0, System.currentTimeMillis()); |
| 85 |
– |
reg.setLastAlertTimeout(0, reg.getAlertTimeout(reg.getLastAlertLevel(0), 0)); |
| 86 |
– |
// -- SEND |
| 87 |
– |
fireAlert(source, timeSinceLastHB, reg, lastAlert); |
| 88 |
– |
} |
| 89 |
– |
// if we don't have a timeout configured...we got STRAIGHT to the next level |
| 90 |
– |
} else { |
| 91 |
– |
int lastAlert = reg.getLastAlertLevel(0); |
| 92 |
– |
reg.escalateAlert(0); |
| 93 |
– |
reg.setTimeLastSent(0, System.currentTimeMillis()); |
| 94 |
– |
reg.setLastAlertTimeout(0, reg.getAlertTimeout(reg.getLastAlertLevel(0), 0)); |
| 95 |
– |
// -- SEND |
| 96 |
– |
fireAlert(source, timeSinceLastHB, reg, lastAlert); |
| 97 |
– |
} |
| 98 |
– |
|
| 99 |
– |
// we must be on ok, check the timeout value for this |
| 100 |
– |
} else { |
| 101 |
– |
// if we were on an OK alert before, then we don't do anything |
| 102 |
– |
// but if we weren't we only set OK, once the timeout of the last |
| 103 |
– |
// alert has occourd |
| 104 |
– |
if (reg.getLastAlertLevel(0) != Alert.alertOK) { |
| 105 |
– |
long timeout = reg.getLastAlertTimeout(0); |
| 106 |
– |
if ((timeout > 0) && (reg.getTimeLastSent(0) > 0)) { |
| 107 |
– |
if ((System.currentTimeMillis() - reg.getTimeLastSent(0)) > timeout) { |
| 108 |
– |
int lastAlert = reg.getLastAlertLevel(0); |
| 109 |
– |
reg.setLastAlertLevel(0, Alert.alertOK); |
| 110 |
– |
reg.setTimeLastSent(0, System.currentTimeMillis()); |
| 111 |
– |
reg.setLastAlertTimeout(0, timeout); |
| 112 |
– |
// -- SEND |
| 113 |
– |
fireAlert(source, timeSinceLastHB, reg, lastAlert); |
| 114 |
– |
} |
| 115 |
– |
} |
| 116 |
– |
} |
| 117 |
– |
} |
| 118 |
– |
} |
| 39 |
|
|
| 40 |
+ |
//---PUBLIC METHODS--- |
| 41 |
+ |
|
| 42 |
|
public void analysePacket(XMLPacket packet) { |
| 43 |
< |
if (packet.getParam("packet.attributes.type").equals("heartbeat")) { |
| 44 |
< |
String source = packet.getParam("packet.attributes.machine_name"); |
| 45 |
< |
if (!_hostsHB.containsKey(source)) { |
| 46 |
< |
_hostsReg.put(source, new Register(source, _name, 1)); |
| 47 |
< |
_hostsHB.put(source, new HeartbeatHolder()); |
| 43 |
> |
String source = packet.getParam("packet.attributes.machine_name"); |
| 44 |
> |
if (!_hosts.containsKey(source)) { |
| 45 |
> |
synchronized(this) { |
| 46 |
> |
HashMap registerHash = new HashMap(); |
| 47 |
> |
registerHash.put(source, new Register(source, _name)); |
| 48 |
> |
_hosts.put(source, new HeartbeatHolder(registerHash)); |
| 49 |
|
} |
| 127 |
– |
HeartbeatHolder lastHeartbeat = (HeartbeatHolder) _hostsReg.get(source); |
| 128 |
– |
lastHeartbeat.setLastHeartbeat((int)System.currentTimeMillis()/1000); |
| 50 |
|
} |
| 51 |
+ |
HeartbeatHolder lastHeartbeat = (HeartbeatHolder) _hosts.get(source); |
| 52 |
+ |
lastHeartbeat.setLastHeartbeat(System.currentTimeMillis()/1000); |
| 53 |
|
} |
| 54 |
|
|
| 55 |
|
/** |
| 56 |
|
* Overrides the {@link java.lang.Object#toString() Object.toString()} |
| 57 |
|
* method to provide clean logging (every class should have this). |
| 58 |
|
* |
| 59 |
< |
* This uses the uk.ac.ukc.iscream.util.NameFormat class |
| 59 |
> |
* This uses the uk.org.iscream.util.NameFormat class |
| 60 |
|
* to format the toString() |
| 61 |
|
* |
| 62 |
|
* @return the name of this class and its CVS revision |
| 77 |
|
|
| 78 |
|
//---PRIVATE METHODS--- |
| 79 |
|
|
| 80 |
< |
private int checkAttributeThreshold(int timeSinceLastHB, Register reg) { |
| 80 |
> |
private int checkAttributeThreshold(long timeSinceLastHB, Register reg) { |
| 81 |
|
for(int thresholdLevel = Alert.thresholdLevels.length - 1; thresholdLevel >= 0; thresholdLevel--) { |
| 82 |
|
if (reg.getThreshold(thresholdLevel) != -1.0) { |
| 83 |
< |
if (reg.getThreshold(thresholdLevel) < timeSinceLastHB) { |
| 83 |
> |
if (((long) reg.getThreshold(thresholdLevel)) < timeSinceLastHB) { |
| 84 |
|
return thresholdLevel; |
| 85 |
|
} |
| 86 |
|
} |
| 87 |
|
} |
| 88 |
< |
return 0; |
| 88 |
> |
return Alert.thresholdNORMAL; |
| 89 |
|
} |
| 167 |
– |
|
| 168 |
– |
private void fireAlert(String source, int timeSinceLastHB, Register reg, int lastAlert) { |
| 169 |
– |
int alertLevel = reg.getLastAlertLevel(0); |
| 170 |
– |
int thresholdLevel = reg.getLastThresholdLevel(0); |
| 171 |
– |
String currentValue = String.valueOf(timeSinceLastHB); |
| 172 |
– |
String attributeName = "Heartbeat"; |
| 173 |
– |
String thresholdValue = String.valueOf(reg.getThreshold(thresholdLevel)); |
| 174 |
– |
String time = Long.toString(reg.getAlertTimeout(reg.getLastAlertLevel(0), 0) / 1000); |
| 175 |
– |
if (thresholdLevel == Alert.thresholdNORMAL) { |
| 176 |
– |
thresholdValue = "-"; |
| 177 |
– |
} |
| 178 |
– |
if (alertLevel == Alert.alertOK) { |
| 179 |
– |
time = "0"; |
| 180 |
– |
} |
| 181 |
– |
Alert alert = new Alert(alertLevel, lastAlert, thresholdLevel, source, thresholdValue, currentValue, attributeName, time); |
| 182 |
– |
_alerterQueue.add(alert); |
| 183 |
– |
_logger.write(toString(), Logger.DEBUG, "Fired alert for source:" + source + " at alert level:" + Alert.alertLevels[alertLevel] + " on:" + attributeName + " for threshold level:" + Alert.thresholdLevels[thresholdLevel] + " at:" + currentValue + " exceeding threshold of:" +thresholdValue + " next alert sent in:" + time + "secs"); |
| 184 |
– |
} |
| 90 |
|
|
| 91 |
|
//---ACCESSOR/MUTATOR METHODS--- |
| 92 |
|
|
| 93 |
+ |
protected Queue getQueue() { |
| 94 |
+ |
return MonitorManager.getInstance().getHeartbeatQueue(); |
| 95 |
+ |
} |
| 96 |
+ |
|
| 97 |
|
//---ATTRIBUTES--- |
| 98 |
|
|
| 99 |
|
/** |
| 106 |
|
* be changed to null for utility classes. |
| 107 |
|
*/ |
| 108 |
|
private String _name = "Heartbeat"; |
| 200 |
– |
|
| 201 |
– |
/** |
| 202 |
– |
* This holds a reference to the |
| 203 |
– |
* system logger that is being used. |
| 204 |
– |
*/ |
| 205 |
– |
private Logger _logger = ReferenceManager.getInstance().getLogger(); |
| 109 |
|
|
| 207 |
– |
private Queue _alerterQueue = ClientMain._alerterQueue; |
| 208 |
– |
|
| 110 |
|
/** |
| 111 |
|
* A reference to the configuration proxy in use |
| 112 |
|
*/ |
| 113 |
|
private ConfigurationProxy _cp = ConfigurationProxy.getInstance(); |
| 114 |
|
|
| 115 |
< |
private HashMap _hostsHB = new HashMap(); |
| 215 |
< |
private HashMap _hostsReg = new HashMap(); |
| 115 |
> |
private HashMap _hosts = new HashMap(); |
| 116 |
|
|
| 117 |
+ |
private Logger _logger = ReferenceManager.getInstance().getLogger(); |
| 118 |
+ |
|
| 119 |
|
//---STATIC ATTRIBUTES--- |
| 120 |
|
|
| 121 |
|
//---INNER CLASSES--- |
| 122 |
|
|
| 123 |
|
private class HeartbeatHolder { |
| 124 |
|
|
| 125 |
< |
public void setLastHeartbeat(int lastHeartbeat) { |
| 125 |
> |
public HeartbeatHolder(HashMap registerHash) { |
| 126 |
> |
_registerHash = registerHash; |
| 127 |
> |
} |
| 128 |
> |
|
| 129 |
> |
public void setLastHeartbeat(long lastHeartbeat) { |
| 130 |
|
_lastHeartbeat = lastHeartbeat; |
| 131 |
|
} |
| 132 |
|
|
| 133 |
< |
public int getLastHeartbeat() { |
| 133 |
> |
public long getLastHeartbeat() { |
| 134 |
|
return _lastHeartbeat; |
| 135 |
|
} |
| 136 |
|
|
| 137 |
< |
private int _lastHeartbeat; |
| 138 |
< |
} |
| 139 |
< |
|
| 137 |
> |
public HashMap getRegisterHash() { |
| 138 |
> |
return _registerHash; |
| 139 |
> |
} |
| 140 |
> |
|
| 141 |
> |
private long _lastHeartbeat; |
| 142 |
> |
private HashMap _registerHash; |
| 143 |
> |
} |
| 144 |
> |
|
| 145 |
> |
private class HeartbeatWorker extends Thread { |
| 146 |
> |
|
| 147 |
> |
public void run() { |
| 148 |
> |
ConfigurationProxy cp = ConfigurationProxy.getInstance(); |
| 149 |
> |
while(true) { |
| 150 |
> |
// this cycle period of this monitor's checks |
| 151 |
> |
int checkPeriod = 0; |
| 152 |
> |
try { |
| 153 |
> |
checkPeriod = Integer.parseInt(cp.getProperty(_name, "Monitor.Heartbeat.checkPeriod")); |
| 154 |
> |
} catch (PropertyNotFoundException e) { |
| 155 |
> |
checkPeriod = DEFAULT_CHECK_PERIOD; |
| 156 |
> |
_logger.write(this.toString(), Logger.WARNING, "Monitor.Heartbeat.checkPeriod value unavailable using default of " + checkPeriod + " seconds"); |
| 157 |
> |
} catch (NumberFormatException e) { |
| 158 |
> |
checkPeriod = DEFAULT_CHECK_PERIOD; |
| 159 |
> |
_logger.write(this.toString(), Logger.WARNING, "Erronous Monitor.Heartbeat.checkPeriod value in configuration using default of " + checkPeriod + " seconds"); |
| 160 |
> |
} |
| 161 |
> |
|
| 162 |
> |
synchronized(this) { |
| 163 |
> |
// perform the checks (use HB hash, although they *should* be the same) |
| 164 |
> |
Iterator i = _hosts.keySet().iterator(); |
| 165 |
> |
while(i.hasNext()) { |
| 166 |
> |
// get host |
| 167 |
> |
String source = (String) i.next(); |
| 168 |
> |
// check it |
| 169 |
> |
boolean remove = analyseHB(source); |
| 170 |
> |
if(remove) { |
| 171 |
> |
i.remove(); |
| 172 |
> |
} |
| 173 |
> |
} |
| 174 |
> |
} |
| 175 |
> |
|
| 176 |
> |
// wait a while |
| 177 |
> |
try {Thread.sleep(checkPeriod * 1000);} catch (InterruptedException e) {} |
| 178 |
> |
} |
| 179 |
> |
} |
| 180 |
> |
|
| 181 |
> |
private boolean analyseHB(String source) { |
| 182 |
> |
ConfigurationProxy cp = ConfigurationProxy.getInstance(); |
| 183 |
> |
HeartbeatHolder hbHolder = (HeartbeatHolder) _hosts.get(source); |
| 184 |
> |
Register reg = (Register) ((HashMap) hbHolder.getRegisterHash()).get(source); |
| 185 |
> |
|
| 186 |
> |
// get host's HB interval (seconds) |
| 187 |
> |
// this should always exist, thus we set to 0 |
| 188 |
> |
int hostHBinterval = 0; |
| 189 |
> |
try { |
| 190 |
> |
hostHBinterval = Integer.parseInt(cp.getProperty("Host."+source, "Host.TCPUpdateTime")); |
| 191 |
> |
} catch (PropertyNotFoundException e) { |
| 192 |
> |
hostHBinterval = 0; |
| 193 |
> |
_logger.write(this.toString(), Logger.WARNING, "TCPUpdateTime value unavailable using default of " + hostHBinterval + " seconds"); |
| 194 |
> |
} catch (NumberFormatException e) { |
| 195 |
> |
hostHBinterval = 0; |
| 196 |
> |
_logger.write(this.toString(), Logger.WARNING, "Erronous TCPUpdateTime value in configuration using default of " + hostHBinterval + " seconds"); |
| 197 |
> |
} |
| 198 |
> |
|
| 199 |
> |
// get host's last HB time (seconds) |
| 200 |
> |
long lastHeartbeat = hbHolder.getLastHeartbeat(); |
| 201 |
> |
// time since last heartbeat (seconds) |
| 202 |
> |
long timeSinceLastHB = (System.currentTimeMillis()/1000) - lastHeartbeat; |
| 203 |
> |
// time since (or until if negative) the expected heartbeat |
| 204 |
> |
long timeSinceExpectedHB = timeSinceLastHB - (long) hostHBinterval; |
| 205 |
> |
|
| 206 |
> |
// best do a check in case the expected heartbeat is in the future |
| 207 |
> |
if(timeSinceExpectedHB < 0) { |
| 208 |
> |
timeSinceExpectedHB = 0; |
| 209 |
> |
} |
| 210 |
> |
|
| 211 |
> |
// find out the threshold level we're at |
| 212 |
> |
int newThreshold = checkAttributeThreshold(timeSinceExpectedHB, reg); |
| 213 |
> |
|
| 214 |
> |
// process the alert |
| 215 |
> |
processAlert(newThreshold, "Heartbeat", reg, source, String.valueOf(timeSinceExpectedHB)); |
| 216 |
> |
|
| 217 |
> |
if(reg.getLastAlertLevel() == Alert.alertFINAL) { |
| 218 |
> |
return true; |
| 219 |
> |
} |
| 220 |
> |
return false; |
| 221 |
> |
} |
| 222 |
> |
} |
| 223 |
|
} |
