Skip to content

kvm: stop a stuck rbd_open from wedging the agent #14199

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bhouse-nexthop wants to merge 5 commits into
base: 4.22
Choose a base branch
from
Open

kvm: stop a stuck rbd_open from wedging the agent #14199

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
180d29d
kvm: bound Ceph operations and release cluster handles deterministically
bhouse-nexthop Sep 18, 2026
9af2d03
kvm: open RBD images read only where they are only read from
bhouse-nexthop Sep 18, 2026
7932d94
kvm: fix review findings in the Ceph bounding and cleanup changes
bhouse-nexthop Sep 18, 2026
5759f20
kvm: make the Ceph operation timeouts opt in
bhouse-nexthop Sep 18, 2026
7f56e8e
kvm: do not let a failed flush or a failed snapshot removal report su…
bhouse-nexthop Sep 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions agent/conf/agent.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,25 @@ iscsi.session.cleanup.enabled=false
# Time, in seconds, to wait before retrying to rebase during the incremental snapshot process.
# incremental.snapshot.retry.rebase.wait=60

# Time, in seconds, that a single Ceph OSD operation may block before it fails. This bounds one operation
# and not a whole request, so a long running copy or flatten is made up of many operations that are each
# well inside the limit.
# The default of 0 leaves the option unset, which is the librados behaviour of waiting forever. An
# operation that never completes parks the agent thread that issued it, and because the agent runs a
# host's storage commands in sequence, every command queued behind it stops with it.
# Setting a limit is recommended. 60 is a reasonable starting point: it is far above a normal operation
# on a healthy cluster, while still bounding one that will never return.
# rados.osd.op.timeout=0

# Time, in seconds, that a single Ceph monitor operation may block before it fails.
# The default of 0 leaves the option unset, which is the librados behaviour of waiting forever.
# 30 is a reasonable starting point.
# rados.mon.op.timeout=0

# Time, in seconds, that the Ceph client may spend connecting to the monitors.
# Set to 0 to leave the option unset and keep the librados default.
# rados.client.mount.timeout=30

# Path to the VDDK library directory for VMware to KVM conversion via VDDK,
# passed to virt-v2v as -io vddk-libdir=<path>
#vddk.lib.dir=
Expand Down
27 changes: 27 additions & 0 deletions agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -914,6 +914,33 @@ public Property<Integer> getWorkers() {
* */
public static final Property<Integer> INCREMENTAL_SNAPSHOT_RETRY_REBASE_WAIT = new Property<>("incremental.snapshot.retry.rebase.wait", 60);

/**
* Time, in seconds, that a single Ceph OSD operation may block before it fails.<br>
* This bounds each operation, not a whole request, so a long running copy or flatten is made up of
* many operations that are each well inside the limit.<br>
* The default of <code>0</code> leaves the option unset, which keeps the librados behaviour of waiting
* forever. A stuck operation then blocks the agent thread that issued it, and with it every storage
* command queued behind it for that host, so setting a limit is recommended.<br>
* Data type: Integer.<br>
* Default value: <code>0</code>
*/
public static final Property<Integer> RADOS_OSD_OP_TIMEOUT = new Property<>("rados.osd.op.timeout", 0);

/**
* Time, in seconds, that a single Ceph monitor operation may block before it fails.<br>
* The default of <code>0</code> leaves the option unset, which keeps the librados behaviour of waiting
* forever.<br>
* Data type: Integer.<br>
* Default value: <code>0</code>
*/
public static final Property<Integer> RADOS_MON_OP_TIMEOUT = new Property<>("rados.mon.op.timeout", 0);

/**
* Time, in seconds, that the Ceph client may spend connecting to the monitors.<br>
* Data type: Integer.<br>
* Default value: <code>30</code>
*/
public static final Property<Integer> RADOS_CLIENT_MOUNT_TIMEOUT = new Property<>("rados.client.mount.timeout", 30);

public static class Property <T>{
private String name;
Expand Down
45 changes: 30 additions & 15 deletions ...n/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import com.cloud.agent.api.BackupSnapshotAnswer;
import com.cloud.agent.api.BackupSnapshotCommand;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.hypervisor.kvm.storage.CephUtil;
import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
Expand Down Expand Up @@ -96,17 +97,17 @@ public Answer execute(final BackupSnapshotCommand command, final LibvirtComputin
* cmds.timeout
*/
if (primaryPool.getType() == StoragePoolType.RBD) {
Rados r = null;
IoCTX io = null;
Rbd rbd = null;
RbdImage image = null;
try {
final Rados r = new Rados(primaryPool.getAuthUserName());
r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort());
r.confSet("key", primaryPool.getAuthSecret());
r.confSet("client_mount_timeout", "30");
r.connect();
logger.debug("Successfully connected to Ceph cluster at " + r.confGet("mon_host"));

final IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir());
final Rbd rbd = new Rbd(io);
final RbdImage image = rbd.open(snapshotDisk.getName(), snapshotName);
r = CephUtil.connect(primaryPool.getAuthUserName(), primaryPool.getSourceHost(), primaryPool.getSourcePort(), primaryPool.getAuthSecret());

io = r.ioCtxCreate(primaryPool.getSourceDir());
rbd = new Rbd(io);
// The snapshot is only read from here, so it is opened read only.
image = rbd.openReadOnly(snapshotDisk.getName(), snapshotName);
final File fh = new File(snapshotDestPath);
try(BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(fh));) {
final int chunkSize = 4194304;
Expand All @@ -115,24 +116,38 @@ public Answer execute(final BackupSnapshotCommand command, final LibvirtComputin
while (true) {
final byte[] buf = new byte[chunkSize];
final int bytes = image.read(offset, buf, chunkSize);
if (bytes <= 0) {
if (bytes < 0) {
/*
* rbd_read returns a negative errno rather than throwing. Treating that as
* end of image would store a short backup and report it as a success.
*/
throw new RbdException("Failed to read " + snapshotDisk.getName() + " at offset " + offset, bytes);
}
if (bytes == 0) {
break;
}
bos.write(buf, 0, bytes);
offset += bytes;
}
logger.debug("Completed backing up RBD snapshot " + snapshotName + " to " + snapshotDestPath + ". Bytes written: " + toHumanReadableSize(offset));
}catch(final IOException ex)
{
logger.error("BackupSnapshotAnswer:Exception:"+ ex.getMessage());
} catch (final IOException ex) {
/*
* A failed read or write leaves a short file on secondary storage. Reporting success
* here would record a backup that cannot be restored from.
*/
logger.error("Failed to back up " + snapshotDisk.getName() + " to " + snapshotDestPath + ". The error was: " + ex.getMessage(), ex);
return new BackupSnapshotAnswer(command, false, ex.toString(), null, true);
}
r.ioCtxDestroy(io);
} catch (final RadosException e) {
logger.error("A RADOS operation failed. The error was: " + e.getMessage());
return new BackupSnapshotAnswer(command, false, e.toString(), null, true);
} catch (final RbdException e) {
logger.error("A RBD operation on " + snapshotDisk.getName() + " failed. The error was: " + e.getMessage());
return new BackupSnapshotAnswer(command, false, e.toString(), null, true);
} finally {
CephUtil.closeQuietly(rbd, image, snapshotDisk.getName());
CephUtil.ioCtxDestroyQuietly(r, io);
CephUtil.shutDownQuietly(r);
}
} else {
final Script scriptCommand = new Script(manageSnapshotPath, cmdsTimeout, logger);
Expand Down
46 changes: 33 additions & 13 deletions ...n/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtManageSnapshotCommandWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,14 @@
import com.ceph.rados.IoCTX;
import com.ceph.rados.Rados;
import com.ceph.rbd.Rbd;
import com.ceph.rbd.RbdException;
import com.ceph.rbd.RbdImage;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.ManageSnapshotAnswer;
import com.cloud.agent.api.ManageSnapshotCommand;
import com.cloud.agent.api.to.StorageFilerTO;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.hypervisor.kvm.storage.CephUtil;
import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
Expand All @@ -48,6 +50,8 @@
@ResourceWrapper(handles = ManageSnapshotCommand.class)
public final class LibvirtManageSnapshotCommandWrapper extends CommandWrapper<ManageSnapshotCommand, Answer, LibvirtComputingResource> {

/** librados reports a missing object as -ENOENT. */
private static final int RBD_ENOENT = -2;

@Override
public Answer execute(final ManageSnapshotCommand command, final LibvirtComputingResource libvirtComputingResource) {
Expand Down Expand Up @@ -113,17 +117,16 @@ public Answer execute(final ManageSnapshotCommand command, final LibvirtComputin
* cord out of a running machine.
*/
if (primaryPool.getType() == StoragePoolType.RBD) {
Rados r = null;
IoCTX io = null;
Rbd rbd = null;
RbdImage image = null;
try {
final Rados r = new Rados(primaryPool.getAuthUserName());
r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort());
r.confSet("key", primaryPool.getAuthSecret());
r.confSet("client_mount_timeout", "30");
r.connect();
logger.debug("Successfully connected to Ceph cluster at " + r.confGet("mon_host"));
r = CephUtil.connect(primaryPool.getAuthUserName(), primaryPool.getSourceHost(), primaryPool.getSourcePort(), primaryPool.getAuthSecret());

final IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir());
final Rbd rbd = new Rbd(io);
final RbdImage image = rbd.open(disk.getName());
io = r.ioCtxCreate(primaryPool.getSourceDir());
rbd = new Rbd(io);
image = rbd.open(disk.getName());

if (command.getCommandSwitch().equalsIgnoreCase(ManageSnapshotCommand.CREATE_SNAPSHOT)) {
logger.debug("Attempting to create RBD snapshot " + disk.getName() + "@" + snapshotName);
Expand All @@ -132,11 +135,28 @@ public Answer execute(final ManageSnapshotCommand command, final LibvirtComputin
logger.debug("Attempting to remove RBD snapshot " + disk.getName() + "@" + snapshotName);
image.snapRemove(snapshotName);
}

rbd.close(image);
r.ioCtxDestroy(io);
} catch (final RbdException e) {
if (ManageSnapshotCommand.DESTROY_SNAPSHOT.equalsIgnoreCase(command.getCommandSwitch()) && e.getReturnValue() == RBD_ENOENT) {
/*
* Already gone. A delete whose end state is "the snapshot is not there" has
* succeeded, and failing here would break a retried delete.
*/
logger.info("RBD snapshot " + disk.getName() + "@" + snapshotName + " was already gone.");
} else {
/*
* Reporting success here would record a snapshot in CloudStack that does not
* exist on the cluster, or drop one that is still there.
*/
logger.error("A RBD snapshot operation on " + disk.getName() + " failed. The error was: " + e.getMessage(), e);
return new ManageSnapshotAnswer(command, false, "Failed to manage snapshot: " + e.toString());
}
} catch (final Exception e) {
logger.error("A RBD snapshot operation on " + disk.getName() + " failed. The error was: " + e.getMessage());
logger.error("A RBD snapshot operation on " + disk.getName() + " failed. The error was: " + e.getMessage(), e);
return new ManageSnapshotAnswer(command, false, "Failed to manage snapshot: " + e.toString());
} finally {
CephUtil.closeQuietly(rbd, image, disk.getName());
CephUtil.ioCtxDestroyQuietly(r, io);
CephUtil.shutDownQuietly(r);
}
} else {
/* VM is not running, create a snapshot by ourself */
Expand Down
49 changes: 24 additions & 25 deletions ...n/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.hypervisor.kvm.storage.CephUtil;
import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
Expand All @@ -61,11 +62,6 @@
@ResourceWrapper(handles = RevertSnapshotCommand.class)
public class LibvirtRevertSnapshotCommandWrapper extends CommandWrapper<RevertSnapshotCommand, Answer, LibvirtComputingResource> {

private static final String MON_HOST = "mon_host";
private static final String KEY = "key";
private static final String CLIENT_MOUNT_TIMEOUT = "client_mount_timeout";
private static final String RADOS_CONNECTION_TIMEOUT = "30";

protected Set<StoragePoolType> storagePoolTypesThatSupportRevertSnapshot = new HashSet<>(Arrays.asList(StoragePoolType.RBD, StoragePoolType.Filesystem,
StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint));

Expand All @@ -92,26 +88,29 @@ public Answer execute(final RevertSnapshotCommand command, final LibvirtComputin
KVMStoragePool primaryPool = snapshotDisk.getPool();

if (primaryPool.getType() == StoragePoolType.RBD) {
Rados rados = new Rados(primaryPool.getAuthUserName());
rados.confSet(MON_HOST, primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort());
rados.confSet(KEY, primaryPool.getAuthSecret());
rados.confSet(CLIENT_MOUNT_TIMEOUT, RADOS_CONNECTION_TIMEOUT);
rados.connect();

String[] rbdPoolAndVolumeAndSnapshot = snapshotRelPath.split("/");
int snapshotIndex = rbdPoolAndVolumeAndSnapshot.length - 1;
String rbdSnapshotId = rbdPoolAndVolumeAndSnapshot[snapshotIndex];

IoCTX io = rados.ioCtxCreate(primaryPool.getSourceDir());
Rbd rbd = new Rbd(io);

logger.debug(String.format("Attempting to rollback RBD snapshot [name:%s], [volumeid:%s], [snapshotid:%s]", snapshot.getName(), volumePath, rbdSnapshotId));

RbdImage image = rbd.open(volumePath);
image.snapRollBack(rbdSnapshotId);

rbd.close(image);
rados.ioCtxDestroy(io);
Rados rados = null;
IoCTX io = null;
Rbd rbd = null;
RbdImage image = null;
try {
rados = CephUtil.connect(primaryPool.getAuthUserName(), primaryPool.getSourceHost(), primaryPool.getSourcePort(), primaryPool.getAuthSecret());

String[] rbdPoolAndVolumeAndSnapshot = snapshotRelPath.split("/");
int snapshotIndex = rbdPoolAndVolumeAndSnapshot.length - 1;
String rbdSnapshotId = rbdPoolAndVolumeAndSnapshot[snapshotIndex];

io = rados.ioCtxCreate(primaryPool.getSourceDir());
rbd = new Rbd(io);

logger.debug(String.format("Attempting to rollback RBD snapshot [name:%s], [volumeid:%s], [snapshotid:%s]", snapshot.getName(), volumePath, rbdSnapshotId));

image = rbd.open(volumePath);
image.snapRollBack(rbdSnapshotId);
} finally {
CephUtil.closeQuietly(rbd, image, volumePath);
CephUtil.ioCtxDestroyQuietly(rados, io);
CephUtil.shutDownQuietly(rados);
}
} else {
if (snapshotImageStore != null && DataStoreRole.Primary != snapshotImageStore.getRole()) {
secondaryStoragePool = storagePoolMgr.getStoragePoolByURI(snapshotImageStore.getUrl());
Expand Down
Loading
Loading