Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.alert;

import com.cloud.dc.DataCenter;
import com.cloud.dc.Pod;
import com.cloud.host.Host;

/**
* Shared formatting for the host/zone/pod description that recurs, independently
* hand-rolled and inconsistently worded (and occasionally mislabelled), across the
* HA and agent-management alert call sites. See CLOUDSTACK-7297.
*/
Comment thread
DaanHoogland marked this conversation as resolved.
public final class AlertFormatUtils {

private AlertFormatUtils() {
}

public static String describeHostLocation(Host host, DataCenter zone, Pod pod) {
return String.format("name: %s (id: %d, uuid: %s), availability zone: %s, pod: %s",
host.getName(), host.getId(), host.getUuid(),
zone != null ? zone.getName() : "unknown",
pod != null ? pod.getName() : "unknown");
}
Comment on lines +33 to +38
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.alert;

import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.when;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;

import com.cloud.dc.DataCenter;
import com.cloud.dc.Pod;
import com.cloud.host.Host;

@RunWith(MockitoJUnitRunner.class)
public class AlertFormatUtilsTest {

@Mock
Host host;
@Mock
DataCenter zone;
@Mock
Pod pod;

@Test
public void describeHostLocationIncludesNameIdUuidZoneAndPod() {
setUpHost();
setUpZone();
setUpPod();

String result = AlertFormatUtils.describeHostLocation(host, zone, pod);

assertEquals("name: cs-kvm06 (id: 37, uuid: host-uuid), availability zone: Milton1, pod: Milton1-Pod1", result);
}

@Test
public void describeHostLocationFallsBackToUnknownForNullZone() {
setUpHost();
setUpPod();

String result = AlertFormatUtils.describeHostLocation(host, null, pod);

assertEquals("name: cs-kvm06 (id: 37, uuid: host-uuid), availability zone: unknown, pod: Milton1-Pod1", result);
}

@Test
public void describeHostLocationFallsBackToUnknownForNullPod() {
setUpHost();
setUpZone();

String result = AlertFormatUtils.describeHostLocation(host, zone, null);

assertEquals("name: cs-kvm06 (id: 37, uuid: host-uuid), availability zone: Milton1, pod: unknown", result);
}

@Test
public void describeHostLocationFallsBackToUnknownForNullZoneAndPod() {
setUpHost();

String result = AlertFormatUtils.describeHostLocation(host, null, null);

assertEquals("name: cs-kvm06 (id: 37, uuid: host-uuid), availability zone: unknown, pod: unknown", result);
}

private void setUpHost() {
when(host.getName()).thenReturn("cs-kvm06");
when(host.getId()).thenReturn(37L);
when(host.getUuid()).thenReturn("host-uuid");
}

private void setUpZone() {
when(zone.getName()).thenReturn("Milton1");
}

private void setUpPod() {
when(pod.getName()).thenReturn("Milton1-Pod1");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@
import com.cloud.agent.api.UnsupportedAnswer;
import com.cloud.agent.transport.Request;
import com.cloud.agent.transport.Response;
import com.cloud.alert.AlertFormatUtils;
import com.cloud.alert.AlertManager;
import com.cloud.cluster.ManagementServerHostVO;
import com.cloud.cluster.dao.ManagementServerHostDao;
Expand Down Expand Up @@ -1151,7 +1152,7 @@ protected boolean handleDisconnectWithInvestigation(final AgentAttache attache,
logger.debug(String.format("Skipping sending alert for %s as it already in %s state",
host, host.getStatus()));
} else if (!HOST_DOWN_ALERT_UNSUPPORTED_HOST_TYPES.contains(host.getType())) {
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Host down, " + host.getId(), message);
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Host down, " + host, message);
}
event = Status.Event.HostDown;
} else if (determinedState == Status.Up) {
Expand All @@ -1173,7 +1174,7 @@ protected boolean handleDisconnectWithInvestigation(final AgentAttache attache,
} else if (currentStatus == Status.Up) {
final DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());
final HostPodVO podVO = _podDao.findById(host.getPodId());
final String hostDesc = "name: " + host.getName() + " (id:" + host.getUuid() + "), availability zone: " + dcVO.getName() + ", pod: " + podVO.getName();
final String hostDesc = AlertFormatUtils.describeHostLocation(host, dcVO, podVO);
if (host.getType() != Host.Type.SecondaryStorage && host.getType() != Host.Type.ConsoleProxy) {
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Host disconnected, " + hostDesc,
"If the agent for host [" + hostDesc + "] is not restarted within " + AlertWait + " seconds, host will go to Alert state");
Expand All @@ -1184,12 +1185,11 @@ protected boolean handleDisconnectWithInvestigation(final AgentAttache attache,
// if we end up here we are in alert state, send an alert
final DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());
final HostPodVO podVO = _podDao.findById(host.getPodId());
final String podName = podVO != null ? podVO.getName() : "NO POD";
final String hostDesc = String.format("%s, availability zone: %s, pod: %s", host, dcVO, podName);
final String hostDesc = AlertFormatUtils.describeHostLocation(host, dcVO, podVO);
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST,
host.getDataCenterId(), host.getPodId(),
String.format("Host in ALERT state, %s", hostDesc),
String.format("In availability zone %s, host is in alert state: %s", dcVO, host));
String.format("Host is in alert state: %s", hostDesc));
}
} else {
logger.debug("The next status of agent {} is not Alert, no need to investigate what happened", host);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@
import com.cloud.agent.api.routing.NetworkElementCommand;
import com.cloud.agent.api.to.NicTO;
import com.cloud.agent.api.to.deployasis.OVFNetworkTO;
import com.cloud.alert.AlertFormatUtils;
import com.cloud.alert.AlertManager;
import com.cloud.api.query.dao.DomainRouterJoinDao;
import com.cloud.api.query.vo.DomainRouterJoinVO;
Expand Down Expand Up @@ -4497,7 +4498,8 @@ public void processConnect(final Host host, final StartupCommand cmd, final bool

if (!answer.getResult()) {
logger.warn("Unable to setup agent {} due to {}", host, answer.getDetails());
final String msg = "Incorrect Network setup on agent, Reinitialize agent after network names are setup, details : " + answer.getDetails();
final String msg = "Incorrect Network setup on agent " + AlertFormatUtils.describeHostLocation(host, dc, null)
+ ", Reinitialize agent after network names are setup, details : " + answer.getDetails();
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, dcId, host.getPodId(), msg, msg);
throw new ConnectionException(true, msg);
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) {
vmSnapshotHelper.vmSnapshotStateTransitTo(vmSnapshot, VMSnapshot.Event.OperationFailed);

String subject = "Take snapshot failed for Instance: " + userVm.getDisplayName();
String message = "Snapshot operation failed for Instance: " + userVm.getDisplayName() + ", Please check and delete if any stale volumes created with Instance Snapshot id: " + vmSnapshot.getVmId();
String message = "Snapshot operation failed for Instance: " + userVm.getDisplayName() + ", Please check and delete if any stale volumes created with " + vmSnapshot;
alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_VM_SNAPSHOT, userVm.getDataCenterId(), userVm.getPodIdToDeployIn(), subject, message);
} catch (NoTransitionException e1) {
logger.error("Cannot set Instance Snapshot state due to: " + e1.getMessage());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher<? extends Bas
result.setSuccess(false);
result.setResult(answer.getErrorString());
caller.complete(result);
String msg = "Failed to register template: " + obj.getUuid() + " with error: " + answer.getErrorString();
String msg = "Failed to register template: " + obj + " with error: " + answer.getErrorString();
Comment thread
DaanHoogland marked this conversation as resolved.
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, _vmTemplateZoneDao.listByTemplateId(obj.getId()).get(0).getZoneId(), null, msg, msg);
logger.error(msg);
} else if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
Expand Down Expand Up @@ -306,7 +306,7 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher<? extends Bas
result.setSuccess(false);
result.setResult(answer.getErrorString());
caller.complete(result);
String msg = "Failed to upload volume: " + obj.getUuid() + " with error: " + answer.getErrorString();
String msg = "Failed to upload volume: " + obj + " with error: " + answer.getErrorString();
Comment thread
DaanHoogland marked this conversation as resolved.
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED,
(volStoreVO == null ? -1L : volStoreVO.getZoneId()), null, msg, msg);
logger.error(msg);
Expand Down Expand Up @@ -352,7 +352,7 @@ protected Void createSnapshotAsyncCallback(AsyncCallbackDispatcher<? extends Bas
result.setSuccess(false);
result.setResult(answer.getErrorString());
caller.complete(result);
String msg = "Failed to copy snapshot: " + obj.getUuid() + " with error: " + answer.getErrorString();
String msg = "Failed to copy snapshot: " + obj + " with error: " + answer.getErrorString();
Comment thread
DaanHoogland marked this conversation as resolved.
Long zoneId = dataStoreManager.getStoreZoneId(store.getId(), store.getRole());
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED,
zoneId, null, msg, msg);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ public boolean hostConnect(long hostId, long poolId) throws StorageConflictExcep
}

if (!answer.getResult()) {
String msg = String.format("Unable to attach storage pool %s to the host %d", pool, hostId);
String msg = String.format("Unable to attach storage pool %s to the host %s", pool, host);
alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, pool.getDataCenterId(), pool.getPodId(), msg, msg);
throw new CloudRuntimeException(String.format("Unable to establish connection from storage head to storage pool %s due to %s %s",
pool, answer.getDetails(), pool.getUuid()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,8 @@ private void sendModifyStoragePoolCommand(ModifyStoragePoolCommand cmd, StorageP
}

if (!answer.getResult()) {
String msg = String.format("Unable to attach storage pool %s to host %d", storagePool, hostId);
HostVO host = _hostDao.findById(hostId);
String msg = String.format("Unable to attach storage pool %s to host %s", storagePool, host);

_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, storagePool.getDataCenterId(), storagePool.getPodId(), msg, msg);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public boolean hostConnect(long hostId, long poolId) {
}

if (!answer.getResult()) {
String msg = String.format("Unable to attach storage pool %s to host %d", pool, hostId);
String msg = String.format("Unable to attach storage pool %s to host %s", pool, host);

_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, pool.getDataCenterId(), pool.getPodId(), msg, msg);

Expand All @@ -107,8 +107,8 @@ public boolean hostConnect(long hostId, long poolId) {

if (!(answer instanceof ModifyStoragePoolAnswer)) {
throw new CloudRuntimeException(String.format(
"Unexpected answer type %s returned for modify storage pool command for pool %s on host %d",
answer.getClass().getName(), pool, hostId));
"Unexpected answer type %s returned for modify storage pool command for pool %s on host %s",
answer.getClass().getName(), pool, host));
}

ModifyStoragePoolAnswer mspAnswer = (ModifyStoragePoolAnswer) answer;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1552,7 +1552,7 @@ public boolean canDisconnectHostFromStoragePool(Host host, StoragePool pool) {
final ScaleIOGatewayClient client = getScaleIOClient(pool);
return client.listVolumesMappedToSdc(sdcId).isEmpty();
} catch (Exception e) {
logger.warn("Unable to check whether the host: " + host.getId() + " can be disconnected from storage pool: " + pool.getId() + ", due to " + e.getMessage(), e);
logger.warn("Unable to check whether the host: " + host + " can be disconnected from storage pool: " + pool + ", due to " + e.getMessage(), e);
return false;
}
}
Expand All @@ -1564,7 +1564,7 @@ private void alertHostSdcDisconnection(Host host) {

logger.warn("SDC not connected on the host: {}", host);
String msg = String.format("SDC not connected on the host: %s, reconnect the SDC to MDM", host);
alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "SDC disconnected on host: " + host.getUuid(), msg);
alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "SDC disconnected on host: " + host, msg);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ private String getSdcIdOfHost(HostVO host, DataStore dataStore) {
if (MapUtils.isEmpty(poolDetails)) {
String msg = String.format("PowerFlex storage SDC details not found on the host: %s, (re)install SDC and restart agent", host);
logger.warn(msg);
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "SDC details not found on host: " + host.getUuid(), msg);
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "SDC details not found on host: " + host, msg);
return null;
}

Expand All @@ -138,16 +138,16 @@ private String getSdcIdOfHost(HostVO host, DataStore dataStore) {
if (StringUtils.isBlank(sdcId)) {
String msg = String.format("Couldn't retrieve PowerFlex storage SDC details from the host: %s, add MDMs if On-demand connect disabled or try (re)install SDC & restart agent", host);
logger.warn(msg);
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "SDC details not found on host: " + host.getUuid(), msg);
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "SDC details not found on host: " + host, msg);
return null;
}

if (details.containsKey(ScaleIOSDCManager.ConnectOnDemand.key())) {
String connectOnDemand = details.get(ScaleIOSDCManager.ConnectOnDemand.key());
if (connectOnDemand != null && !Boolean.parseBoolean(connectOnDemand) && !_sdcManager.isHostSdcConnected(sdcId, dataStore, 15)) {
logger.warn("SDC not connected on the host: " + hostId);
String msg = "SDC not connected on the host: " + hostId + ", reconnect the SDC to MDM and restart agent";
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "SDC not connected on host: " + host.getUuid(), msg);
logger.warn("SDC not connected on the host: " + host);
String msg = "SDC not connected on host: " + host + ", reconnect the SDC to MDM and restart agent";
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "SDC not connected on host: " + host, msg);
return null;
}
}
Expand Down Expand Up @@ -213,15 +213,15 @@ public boolean hostDisconnected(long hostId, long poolId) {
ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(false, storagePool, storagePool.getPath(), details);
ModifyStoragePoolAnswer answer = sendModifyStoragePoolCommand(cmd, storagePool, host);
if (!answer.getResult()) {
logger.error("Failed to disconnect storage pool: " + storagePool + " and host: " + hostId);
logger.error("Failed to disconnect storage pool: " + storagePool + " and host: " + host);
return false;
}

StoragePoolHostVO storagePoolHost = _storagePoolHostDao.findByPoolHost(poolId, hostId);
if (storagePoolHost != null) {
_storagePoolHostDao.deleteStoragePoolHostDetails(hostId, poolId);
}
logger.info("Connection removed between storage pool: " + storagePool + " and host: " + hostId);
logger.info("Connection removed between storage pool: " + storagePool + " and host: " + host);
return true;
}

Expand Down
Loading
Loading