Visitar URL original
Allow limited pool recovery in clusters by stgraber · Pull Request #3886 · lxc/incus · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions cmd/incus/admin_recover.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,18 @@ func (c *cmdAdminRecover) run(cmd *cobra.Command, args []string) error {
unknownPools := make([]api.StoragePoolsPost, 0, len(existingPools))

// Build up a list of unknown pools to scan.
// We don't offer this option if the server is clustered because we don't allow creating storage pools on
// an individual server when clustered.
if !isClustered {
var supportedDriverNames []string
// When clustered, we only offer shared storage pools because we don't allow creating
// storage pools on an individual server when clustered.
var supportedDriverNames []string
for _, supportedDriver := range server.Environment.StorageSupportedDrivers {
if isClustered && !supportedDriver.Remote {
continue
}

supportedDriverNames = append(supportedDriverNames, supportedDriver.Name)
}

if len(supportedDriverNames) > 0 {
for {
addUnknownPool, err := c.global.asker.AskBool(i18n.G("Would you like to recover another storage pool?")+" (yes/no) [default=no]: ", "no")
if err != nil {
Expand All @@ -81,13 +88,6 @@ func (c *cmdAdminRecover) run(cmd *cobra.Command, args []string) error {
break
}

// Get available storage drivers if not done already.
if supportedDriverNames == nil {
for _, supportedDriver := range server.Environment.StorageSupportedDrivers {
supportedDriverNames = append(supportedDriverNames, supportedDriver.Name)
}
}

unknownPool := api.StoragePoolsPost{
StoragePoolPut: api.StoragePoolPut{
Config: make(map[string]string),
Expand Down
82 changes: 64 additions & 18 deletions cmd/incusd/api_internal_recover.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,12 +158,6 @@ func internalRecoverScan(ctx context.Context, s *state.State, userPools []api.St
return response.SmartError(fmt.Errorf("Failed loading existing pool %q: %w", p.Name, err))
}

// If the pool DB record doesn't exist, and we are clustered, then don't proceed
// any further as we do not support pool DB record recovery when clustered.
if s.ServerClustered {
return response.BadRequest(errors.New("Storage pool recovery not supported when clustered"))
}

// If pool doesn't exist in DB, initialize a temporary pool with the supplied info.
poolInfo := api.StoragePool{
Name: p.Name,
Expand All @@ -177,6 +171,12 @@ func internalRecoverScan(ctx context.Context, s *state.State, userPools []api.St
return response.SmartError(fmt.Errorf("Failed to initialize unknown pool %q: %w", p.Name, err))
}

// When clustered, we don't support per-member pool DB record recovery,
// so only allow shared storage pools to be recovered.
if s.ServerClustered && !pool.Driver().Info().Remote {
return response.BadRequest(fmt.Errorf("Only shared storage pools can be recovered when clustered, pool %q isn't shared", p.Name))
}

// Populate configuration with default values.
err := pool.Driver().FillConfig()
if err != nil {
Expand Down Expand Up @@ -346,29 +346,75 @@ func internalRecoverScan(ctx context.Context, s *state.State, userPools []api.St
}
}

var poolName, poolDescription, poolDriverName string
var poolConfig map[string]string

if instPoolVol != nil {
// Create storage pool DB record from config in the instance.
logger.Info("Creating storage pool DB record from instance config", logger.Ctx{"name": instPoolVol.Pool.Name, "description": instPoolVol.Pool.Description, "driver": instPoolVol.Pool.Driver, "config": instPoolVol.Pool.Config})
poolID, err = dbStoragePoolCreateAndUpdateCache(ctx, s, instPoolVol.Pool.Name, instPoolVol.Pool.Description, instPoolVol.Pool.Driver, instPoolVol.Pool.Config)
if err != nil {
return response.SmartError(fmt.Errorf("Failed creating storage pool %q database entry: %w", pool.Name(), err))
}
poolName = instPoolVol.Pool.Name
poolDescription = instPoolVol.Pool.Description
poolDriverName = instPoolVol.Pool.Driver
poolConfig = instPoolVol.Pool.Config
logger.Info("Creating storage pool DB record from instance config", logger.Ctx{"name": poolName, "description": poolDescription, "driver": poolDriverName, "config": poolConfig})
} else {
// Create storage pool DB record from config supplied by user if not
// instance volume pool config found.
poolDriverName := pool.Driver().Info().Name
poolDriverConfig := pool.Driver().Config()
logger.Info("Creating storage pool DB record from user config", logger.Ctx{"name": pool.Name(), "driver": poolDriverName, "config": poolDriverConfig})
poolID, err = dbStoragePoolCreateAndUpdateCache(ctx, s, pool.Name(), "", poolDriverName, poolDriverConfig)
if err != nil {
return response.SmartError(fmt.Errorf("Failed creating storage pool %q database entry: %w", pool.Name(), err))
}
poolName = pool.Name()
poolDriverName = pool.Driver().Info().Name
poolConfig = pool.Driver().Config()
logger.Info("Creating storage pool DB record from user config", logger.Ctx{"name": poolName, "driver": poolDriverName, "config": poolConfig})
}

poolID, err = dbStoragePoolCreateAndUpdateCache(ctx, s, poolName, poolDescription, poolDriverName, poolConfig)
if err != nil {
return response.SmartError(fmt.Errorf("Failed creating storage pool %q database entry: %w", pool.Name(), err))
}

reverter.Add(func() {
_ = dbStoragePoolDeleteAndUpdateCache(context.Background(), s, pool.Name())
})

// When clustered, add the pool to the other cluster members.
// This is only reached for shared storage pools which use the same
// member-specific config on every member.
if s.ServerClustered {
err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error {
members, err := tx.GetNodes(ctx)
if err != nil {
return err
}

// Extract the member-specific config keys.
memberConfig := map[string]string{}
for _, key := range db.NodeSpecificStorageConfig(poolDriverName) {
if poolConfig[key] != "" {
memberConfig[key] = poolConfig[key]
}
}

for _, member := range members {
if member.ID == tx.GetNodeID() {
continue
}

err = tx.UpdateStoragePoolAfterNodeJoin(poolID, member.ID)
if err != nil {
return err
}

err = tx.CreateStoragePoolConfig(poolID, member.ID, memberConfig)
if err != nil {
return err
}
}

return nil
})
if err != nil {
return response.SmartError(fmt.Errorf("Failed adding storage pool %q to cluster members: %w", pool.Name(), err))
}
}

// Set storage pool node to storagePoolCreated.
// Must come before storage pool is loaded from the database.
err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error {
Expand Down
3 changes: 3 additions & 0 deletions doc/howto/disaster_recovery.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ The tool must be run interactively and cannot be used in automated scripts.
When you run the tool, it scans all storage pools that still exist in the database, looking for missing volumes that can be recovered.
You can also specify the details of any unknown storage pools (those that exist on disk but do not exist in the database), and the tool attempts to scan those too.

On a clustered server, only shared storage pools (for example, `ceph` or `lvmcluster`) can be specified as unknown storage pools.
It is up to you to ensure that the same shared storage is available to all cluster members.

After mounting the specified storage pools (if not already mounted), the tool scans them for unknown volumes that look like they are associated with Incus.
Incus maintains a `backup.yaml` file in each instance's storage volume, which contains all necessary information to recover a given instance (including instance configuration, attached devices, storage volume, and pool configuration).
This data can be used to rebuild the instance, storage volume, and storage pool database records.
Expand Down
Loading