Visitar URL original
CLOUDSTACK-7946: · enterdevstudio/cloudstack@d856a2a · GitHub
Skip to content

Commit d856a2a

Browse files
sudisonsudison
committed
CLOUDSTACK-7946:
remove leftover state in volume and snapshot table in case of mgt server shutdown during storage operation. Reviewed-by: Min
1 parent 5018db8 commit d856a2a

17 files changed

Lines changed: 147 additions & 20 deletions

File tree

‎engine/api/src/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323

2424
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
2525
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
26+
import org.apache.cloudstack.framework.config.ConfigKey;
2627

2728
import com.cloud.agent.api.to.VirtualMachineTO;
2829
import com.cloud.dc.DataCenter;
@@ -45,7 +46,6 @@
4546
import com.cloud.vm.DiskProfile;
4647
import com.cloud.vm.VirtualMachine;
4748
import com.cloud.vm.VirtualMachineProfile;
48-
import org.apache.cloudstack.framework.config.ConfigKey;
4949

5050
/**
5151
* VolumeOrchestrationService is a PURE orchestration service on CloudStack
@@ -85,6 +85,8 @@ VolumeInfo moveVolume(VolumeInfo volume, long destPoolDcId, Long destPoolPodId,
8585

8686
Volume migrateVolume(Volume volume, StoragePool destPool) throws StorageUnavailableException;
8787

88+
void cleanupStorageJobs();
89+
8890
void destroyVolume(Volume volume);
8991

9092
DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Long size, Long minIops, Long maxIops, VirtualMachine vm, VirtualMachineTemplate template, Account owner);

‎engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,6 @@ public interface SnapshotService {
2727
boolean revertSnapshot(Long snapshotId);
2828

2929
void syncVolumeSnapshotsToRegionStore(long volumeId, DataStore store);
30+
31+
void cleanupVolumeDuringSnapshotFailure(Long volumeId, Long snapshotId);
3032
}

server/src/com/cloud/storage/VmWorkAttachVolume.java renamed to engine/components-api/src/com/cloud/vm/VmWorkAttachVolume.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,7 @@
1414
// KIND, either express or implied. See the License for the
1515
// specific language governing permissions and limitations
1616
// under the License.
17-
package com.cloud.storage;
18-
19-
import com.cloud.vm.VmWork;
17+
package com.cloud.vm;
2018

2119
public class VmWorkAttachVolume extends VmWork {
2220
private static final long serialVersionUID = 553291814854451740L;

server/src/com/cloud/storage/VmWorkDetachVolume.java renamed to engine/components-api/src/com/cloud/vm/VmWorkDetachVolume.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,7 @@
1414
// KIND, either express or implied. See the License for the
1515
// specific language governing permissions and limitations
1616
// under the License.
17-
package com.cloud.storage;
18-
19-
import com.cloud.vm.VmWork;
17+
package com.cloud.vm;
2018

2119
public class VmWorkDetachVolume extends VmWork {
2220
private static final long serialVersionUID = -8722243207385263101L;

server/src/com/cloud/storage/VmWorkMigrateVolume.java renamed to engine/components-api/src/com/cloud/vm/VmWorkMigrateVolume.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,7 @@
1414
// KIND, either express or implied. See the License for the
1515
// specific language governing permissions and limitations
1616
// under the License.
17-
package com.cloud.storage;
18-
19-
import com.cloud.vm.VmWork;
17+
package com.cloud.vm;
2018

2119
public class VmWorkMigrateVolume extends VmWork {
2220
private static final long serialVersionUID = -565778516928408602L;

server/src/com/cloud/storage/VmWorkResizeVolume.java renamed to engine/components-api/src/com/cloud/vm/VmWorkResizeVolume.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,7 @@
1414
// KIND, either express or implied. See the License for the
1515
// specific language governing permissions and limitations
1616
// under the License.
17-
package com.cloud.storage;
18-
19-
import com.cloud.vm.VmWork;
17+
package com.cloud.vm;
2018

2119
public class VmWorkResizeVolume extends VmWork {
2220
private static final long serialVersionUID = 6112366316907642498L;

server/src/com/cloud/storage/VmWorkTakeVolumeSnapshot.java renamed to engine/components-api/src/com/cloud/vm/VmWorkTakeVolumeSnapshot.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,7 @@
1414
// KIND, either express or implied. See the License for the
1515
// specific language governing permissions and limitations
1616
// under the License.
17-
package com.cloud.storage;
18-
19-
import com.cloud.vm.VmWork;
17+
package com.cloud.vm;
2018

2119
public class VmWorkTakeVolumeSnapshot extends VmWork {
2220

‎engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -563,6 +563,7 @@ public boolean start() {
563563
_executor.scheduleAtFixedRate(new TransitionTask(), VmOpCleanupInterval.value(), VmOpCleanupInterval.value(), TimeUnit.SECONDS);
564564
cancelWorkItems(_nodeId);
565565

566+
volumeMgr.cleanupStorageJobs();
566567
// cleanup left over place holder works
567568
_workJobDao.expungeLeftoverWorkJobs(ManagementServerNode.getManagementServerId());
568569
return true;

‎engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java‎

Lines changed: 75 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@
5353
import org.apache.cloudstack.framework.config.ConfigDepot;
5454
import org.apache.cloudstack.framework.config.ConfigKey;
5555
import org.apache.cloudstack.framework.config.Configurable;
56+
import org.apache.cloudstack.framework.jobs.AsyncJobManager;
57+
import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
5658
import org.apache.cloudstack.storage.command.CommandResult;
5759
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
5860
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
@@ -63,6 +65,7 @@
6365
import com.cloud.agent.api.to.DiskTO;
6466
import com.cloud.agent.api.to.VirtualMachineTO;
6567
import com.cloud.agent.manager.allocator.PodAllocator;
68+
import com.cloud.cluster.ClusterManager;
6669
import com.cloud.configuration.Resource.ResourceType;
6770
import com.cloud.dc.DataCenter;
6871
import com.cloud.dc.Pod;
@@ -116,6 +119,10 @@
116119
import com.cloud.vm.VirtualMachine.State;
117120
import com.cloud.vm.VirtualMachineProfile;
118121
import com.cloud.vm.VirtualMachineProfileImpl;
122+
import com.cloud.vm.VmWorkAttachVolume;
123+
import com.cloud.vm.VmWorkMigrateVolume;
124+
import com.cloud.vm.VmWorkSerializer;
125+
import com.cloud.vm.VmWorkTakeVolumeSnapshot;
119126
import com.cloud.vm.dao.UserVmDao;
120127

121128
public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrationService, Configurable {
@@ -155,6 +162,10 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
155162
SnapshotService _snapshotSrv;
156163
@Inject
157164
protected UserVmDao _userVmDao;
165+
@Inject
166+
protected AsyncJobManager _jobMgr;
167+
@Inject
168+
ClusterManager clusterManager;
158169

159170
private final StateMachine2<Volume.State, Volume.Event, Volume> _volStateMachine;
160171
protected List<StoragePoolAllocator> _storagePoolAllocators;
@@ -1342,9 +1353,71 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
13421353
return true;
13431354
}
13441355

1356+
private void cleanupVolumeDuringAttachFailure(Long volumeId) {
1357+
VolumeVO volume = _volsDao.findById(volumeId);
1358+
if (volume == null) {
1359+
return;
1360+
}
1361+
1362+
if (volume.getState().equals(Volume.State.Creating)) {
1363+
s_logger.debug("Remove volume: " + volume.getId() + ", as it's leftover from last mgt server stop");
1364+
_volsDao.remove(volume.getId());
1365+
}
1366+
}
1367+
1368+
private void cleanupVolumeDuringMigrationFailure(Long volumeId, Long destPoolId) {
1369+
StoragePool destPool = (StoragePool)dataStoreMgr.getDataStore(destPoolId, DataStoreRole.Primary);
1370+
if (destPool == null) {
1371+
return;
1372+
}
1373+
1374+
VolumeVO volume = _volsDao.findById(volumeId);
1375+
if (volume.getState() == Volume.State.Migrating) {
1376+
VolumeVO duplicateVol = _volsDao.findByPoolIdName(destPoolId, volume.getName());
1377+
if (duplicateVol != null) {
1378+
s_logger.debug("Remove volume " + duplicateVol.getId() + " on storage pool " + destPoolId);
1379+
_volsDao.remove(duplicateVol.getId());
1380+
}
1381+
1382+
s_logger.debug("change volume state to ready from migrating in case migration failure for vol: " + volumeId);
1383+
volume.setState(Volume.State.Ready);
1384+
_volsDao.update(volumeId, volume);
1385+
}
1386+
1387+
}
1388+
1389+
private void cleanupVolumeDuringSnapshotFailure(Long volumeId, Long snapshotId) {
1390+
_snapshotSrv.cleanupVolumeDuringSnapshotFailure(volumeId, snapshotId);
1391+
VolumeVO volume = _volsDao.findById(volumeId);
1392+
if (volume.getState() == Volume.State.Snapshotting) {
1393+
s_logger.debug("change volume state back to Ready: " + volume.getId());
1394+
volume.setState(Volume.State.Ready);
1395+
_volsDao.update(volume.getId(), volume);
1396+
}
1397+
}
1398+
13451399
@Override
1346-
public boolean start() {
1347-
return true;
1400+
public void cleanupStorageJobs() {
1401+
//clean up failure jobs related to volume
1402+
List<AsyncJobVO> jobs = _jobMgr.findFailureAsyncJobs(VmWorkAttachVolume.class.getName(),
1403+
VmWorkMigrateVolume.class.getName(), VmWorkTakeVolumeSnapshot.class.getName());
1404+
1405+
for (AsyncJobVO job : jobs) {
1406+
try {
1407+
if (job.getCmd().equalsIgnoreCase(VmWorkAttachVolume.class.getName())) {
1408+
VmWorkAttachVolume work = VmWorkSerializer.deserialize(VmWorkAttachVolume.class, job.getCmdInfo());
1409+
cleanupVolumeDuringAttachFailure(work.getVolumeId());
1410+
} else if (job.getCmd().equalsIgnoreCase(VmWorkMigrateVolume.class.getName())) {
1411+
VmWorkMigrateVolume work = VmWorkSerializer.deserialize(VmWorkMigrateVolume.class, job.getCmdInfo());
1412+
cleanupVolumeDuringMigrationFailure(work.getVolumeId(), work.getDestPoolId());
1413+
} else if (job.getCmd().equalsIgnoreCase(VmWorkTakeVolumeSnapshot.class.getName())) {
1414+
VmWorkTakeVolumeSnapshot work = VmWorkSerializer.deserialize(VmWorkTakeVolumeSnapshot.class, job.getCmdInfo());
1415+
cleanupVolumeDuringSnapshotFailure(work.getVolumeId(), work.getSnapshotId());
1416+
}
1417+
} catch (Exception e) {
1418+
s_logger.debug("clean up job failure, will continue", e);
1419+
}
1420+
}
13481421
}
13491422

13501423
@Override

‎engine/schema/src/com/cloud/storage/dao/VolumeDao.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ public interface VolumeDao extends GenericDao<VolumeVO, Long>, StateDao<Volume.S
6161

6262
List<VolumeVO> findByPoolId(long poolId);
6363

64+
VolumeVO findByPoolIdName(long poolId, String name);
65+
6466
List<VolumeVO> findByPoolId(long poolId, Volume.Type volumeType);
6567

6668
List<VolumeVO> findByInstanceAndDeviceId(long instanceId, long deviceId);

0 commit comments

Comments
 (0)