Merge pull request saltstack#37517 from rallytime/merge-develop

[develop] Merge forward from 2016.11 to develop

doc/topics/cloud/vmware.rst

@@ -588,27 +588,29 @@ Example of a minimal profile:
 Cloning from a Snapshot
 =======================
 
-.. versionadded:: 2016.3.4
 
-Cloning a template works similar to cloning a VM except for the fact that
-a snapshot number must be provided.
+.. versionadded:: 2016.3.5
+
+Cloning from a snapshot requires that one of the
+supported options be set in the cloud profile.
+
+Supported options are ``createNewChildDiskBacking``,
+``moveChildMostDiskBacking``, ``moveAllDiskBackingsAndAllowSharing``
+and ``moveAllDiskBackingsAndDisallowSharing``.
 
 Example of a minimal profile:
 
 .. code-block:: yaml
 
   my-template-clone:
-     provider: vcenter01
-     clonefrom: 'salt_vm'
-     snapshot: 3
-
-.. image:: /_static/snapshot_manager.png
-    :align: center
-    :scale: 70%
-
-.. note::
-    The previous diagram shows how to identify the snapshot number. Selected
-    (third snapshot) is number 3.
+    provider: vcenter01
+    clonefrom: 'salt_vm'
+    snapshot:
+      disk_move_type: createNewChildDiskBacking
+      # these types are also supported
+      # disk_move_type: moveChildMostDiskBacking
+      # disk_move_type: moveAllDiskBackingsAndAllowSharing
+      # disk_move_type: moveAllDiskBackingsAndDisallowSharing
 
 
 Creating a VM
@@ -689,7 +691,7 @@ Example of a complete profile:
 Specifying disk backing mode
 ============================
 
-.. versionadded:: Nitrogen
+.. versionadded:: 2016.3.5
 
 Disk backing mode can now be specified when cloning a VM. This option
 can be set in the cloud profile as shown in example below:
@@ -707,6 +709,7 @@ can be set in the cloud profile as shown in example below:
         disk:
           Hard disk 1:
             mode: 'independent_nonpersistent'
+            size: 42
           Hard disk 2:
             size: 15
             mode: 'independent_nonpersistent'

salt/cloud/clouds/ec2.py

@@ -1135,11 +1135,13 @@ def _get_subnetname_id(subnetname):
         for tag in tags:
             if tag['key'] == 'Name' and tag['value'] == subnetname:
                 log.debug('AWS Subnet ID of {0} is {1}'.format(
-                                        subnetname, subnet['subnetId'] )
+                    subnetname,
+                    subnet['subnetId'])
                 )
                 return subnet['subnetId']
     return None
 
+
 def get_subnetid(vm_):
     '''
     Returns the SubnetId to use
@@ -1157,6 +1159,7 @@ def get_subnetid(vm_):
         return _get_subnetname_id(subnetname)
     return None
 
+
 def _get_securitygroupname_id(securitygroupname_list):
     '''
     Returns the SecurityGroupId of a SecurityGroupName to use
@@ -1175,25 +1178,31 @@ def _get_securitygroupname_id(securitygroupname_list):
             securitygroupid_set.add(sg['groupId'])
     return list(securitygroupid_set)
 
+
 def securitygroupid(vm_):
     '''
     Returns the SecurityGroupId
     '''
     securitygroupid_set = set()
     securitygroupid_list = config.get_cloud_config_value(
-        'securitygroupid', vm_, __opts__, search_global=False
+        'securitygroupid',
+        vm_,
+        __opts__,
+        search_global=False
     )
     if securitygroupid_list:
         if isinstance(securitygroupid_list, list):
-            securitygroupid_set = securitygroupid_set.union( securitygroupid_list )
+            securitygroupid_set = securitygroupid_set.union(securitygroupid_list)
         else:
             securitygroupid_set.add(securitygroupid_list)
 
     securitygroupname_list = config.get_cloud_config_value(
         'securitygroupname', vm_, __opts__, search_global=False
     )
     if securitygroupname_list:
-        securitygroupid_set = securitygroupid_set.union( _get_securitygroupname_id(securitygroupname_list) )
+        securitygroupid_set = securitygroupid_set.union(
+            _get_securitygroupname_id(securitygroupname_list)
+        )
     return list(securitygroupid_set)
 
 

salt/cloud/clouds/vmware.py

@@ -132,11 +132,15 @@
 
 # Import salt cloud libs
 import salt.config as config
-from salt.ext.six.moves import range
 
 # Attempt to import pyVim and pyVmomi libs
 ESX_5_5_NAME_PORTION = 'VMware ESXi 5.5'
 SAFE_ESX_5_5_CONTROLLER_KEY_INDEX = 200
+FLATTEN_DISK_FULL_CLONE = 'moveAllDiskBackingsAndDisallowSharing'
+COPY_ALL_DISKS_FULL_CLONE = 'moveAllDiskBackingsAndAllowSharing'
+CURRENT_STATE_LINKED_CLONE = 'moveChildMostDiskBacking'
+QUICK_LINKED_CLONE = 'createNewChildDiskBacking'
+
 try:
     from pyVmomi import vim
     HAS_PYVMOMI = True
@@ -645,30 +649,39 @@ def _manage_devices(devices, vm=None, container_ref=None, new_vm_name=None):
                     # there is atleast one disk specified to be created/configured
                     unit_number += 1
                     existing_disks_label.append(device.deviceInfo.label)
+                    # log.info('all = %s', str(devices['disk'].keys()))
                     if device.deviceInfo.label in list(devices['disk'].keys()):
                         disk_spec = None
+
                         if 'size' in devices['disk'][device.deviceInfo.label]:
                             size_gb = float(devices['disk'][device.deviceInfo.label]['size'])
                             disk_spec = _get_size_spec(device, size_gb)
+                        else:
+                            raise SaltCloudSystemExit(
+                                'Please specify a size'
+                                ' for the disk "{0}" in'
+                                ' your cloud profile'
+                                ''.format(device.deviceInfo.label))
+
+                        size_kb = int(size_gb * 1024 * 1024)
+                        if device.capacityInKB < size_kb:
+                            # expand the disk
+                            disk_spec = _edit_existing_hard_disk_helper(device, size_kb)
 
                         if 'mode' in devices['disk'][device.deviceInfo.label]:
                             if devices['disk'][device.deviceInfo.label]['mode'] \
                                 in [
                                     'independent_persistent',
-                                    'persistent',
                                     'independent_nonpersistent',
-                                    'nonpersistent',
-                                    'undoable',
-                                    'append'
+                                    'dependent',
                             ]:
                                 mode = devices['disk'][device.deviceInfo.label]['mode']
                                 disk_spec = _get_mode_spec(device, mode, disk_spec)
                             else:
                                 raise SaltCloudSystemExit('Invalid disk'
                                                           ' backing mode'
                                                           ' specified!')
-                        if disk_spec:
-                            device_specs.append(disk_spec)
+                        device_specs.append(disk_spec)
 
             elif isinstance(device.backing, vim.vm.device.VirtualEthernetCard.NetworkBackingInfo) or isinstance(device.backing, vim.vm.device.VirtualEthernetCard.DistributedVirtualPortBackingInfo):
                 # this is a network adapter
@@ -2371,18 +2384,6 @@ def create(vm_):
             raise SaltCloudSystemExit(
                 'The VM/template that you have specified under clonefrom does not exist.'
             )
-
-        snapshot = None
-        if clone_type == 'vm' and 'snapshot' in vm_:
-            num = int(vm_['snapshot']) - 1
-            snapshot = object_ref.rootSnapshot[0]
-            # Drill down to the correct snapshot number
-            for _ in range(num):
-                try:
-                    snapshot = snapshot.childSnapshot[0]
-                except IndexError:
-                    raise SaltCloudSystemExit('Specified snapshot'
-                                              ' does not exist.')
     else:
         clone_type = None
         object_ref = None
@@ -2530,15 +2531,18 @@ def create(vm_):
             config_spec.extraConfig.append(option)
 
     if 'clonefrom' in vm_:
-        # Create the clone specs
-        clone_spec = vim.vm.CloneSpec(
-            template=template,
-            location=reloc_spec,
-            config=config_spec
-        )
-
-        if snapshot:
-            clone_spec.snapshot = snapshot
+        clone_spec = handle_snapshot(
+            config_spec,
+            object_ref,
+            reloc_spec,
+            template,
+            vm_
+        )
+        if not clone_spec:
+            clone_spec = build_clonespec(config_spec,
+                                         object_ref,
+                                         reloc_spec,
+                                         template)
 
         if customization and (devices and 'network' in list(devices.keys())):
             global_ip = vim.vm.customization.GlobalIPSettings()
@@ -2685,6 +2689,75 @@ def create(vm_):
     return data
 
 
+def handle_snapshot(config_spec, object_ref, reloc_spec, template, vm_):
+    '''
+    Returns a clone spec for cloning from shapshots
+    :rtype vim.vm.CloneSpec
+    '''
+    if 'snapshot' not in vm_:
+        return None
+
+    allowed_types = [
+        FLATTEN_DISK_FULL_CLONE,
+        COPY_ALL_DISKS_FULL_CLONE,
+        CURRENT_STATE_LINKED_CLONE,
+        QUICK_LINKED_CLONE,
+    ]
+
+    clone_spec = get_clonespec_for_valid_snapshot(
+        config_spec,
+        object_ref,
+        reloc_spec,
+        template,
+        vm_)
+    if not clone_spec:
+        raise SaltCloudSystemExit('Invalid disk move type specified'
+                                  ' supported types are'
+                                  ' {0}'.format(' '.join(allowed_types)))
+    return clone_spec
+
+
+def get_clonespec_for_valid_snapshot(config_spec, object_ref, reloc_spec, template, vm_):
+    '''
+    return clonespec only if values are valid
+    '''
+    moving = True
+    if QUICK_LINKED_CLONE == vm_['snapshot']['disk_move_type']:
+        reloc_spec.diskMoveType = QUICK_LINKED_CLONE
+    elif CURRENT_STATE_LINKED_CLONE == vm_['snapshot']['disk_move_type']:
+        reloc_spec.diskMoveType = CURRENT_STATE_LINKED_CLONE
+    elif COPY_ALL_DISKS_FULL_CLONE == vm_['snapshot']['disk_move_type']:
+        reloc_spec.diskMoveType = COPY_ALL_DISKS_FULL_CLONE
+    elif FLATTEN_DISK_FULL_CLONE == vm_['snapshot']['disk_move_type']:
+        reloc_spec.diskMoveType = FLATTEN_DISK_FULL_CLONE
+    else:
+        moving = False
+
+    if moving:
+        return build_clonespec(config_spec, object_ref, reloc_spec, template)
+    else:
+        return None
+
+
+def build_clonespec(config_spec, object_ref, reloc_spec, template):
+    '''
+    Returns the clone spec
+    '''
+    if reloc_spec.diskMoveType == QUICK_LINKED_CLONE:
+        return vim.vm.CloneSpec(
+            template=template,
+            location=reloc_spec,
+            config=config_spec,
+            snapshot=object_ref.snapshot.currentSnapshot
+        )
+    else:
+        return vim.vm.CloneSpec(
+            template=template,
+            location=reloc_spec,
+            config=config_spec
+        )
+
+
 def create_datacenter(kwargs=None, call=None):
     '''
     Create a new data center in this VMware environment

salt/modules/dnsmasq.py

@@ -2,14 +2,15 @@
 '''
 Module for managing dnsmasq
 '''
+
+# Import Python libs
 from __future__ import absolute_import
+import logging
+import os
 
 # Import salt libs
 import salt.utils
-
-# Import python libs
-import os
-import logging
+from salt.exceptions import CommandExecutionError
 
 log = logging.getLogger(__name__)
 
@@ -98,20 +99,28 @@ def set_config(config_file='/etc/dnsmasq.conf', follow=True, **kwargs):
             if filename.endswith('#') and filename.endswith('#'):
                 continue
             includes.append('{0}/{1}'.format(dnsopts['conf-dir'], filename))
+
+    ret_kwargs = {}
     for key in kwargs:
+        # Filter out __pub keys as they should not be added to the config file
+        # See Issue #34263 for more information
+        if key.startswith('__'):
+            continue
+        ret_kwargs[key] = kwargs[key]
+
         if key in dnsopts:
             if isinstance(dnsopts[key], str):
                 for config in includes:
                     __salt__['file.sed'](path=config,
-                                    before='^{0}=.*'.format(key),
-                                    after='{0}={1}'.format(key, kwargs[key]))
+                                         before='^{0}=.*'.format(key),
+                                         after='{0}={1}'.format(key, kwargs[key]))
             else:
                 __salt__['file.append'](config_file,
-                                    '{0}={1}'.format(key, kwargs[key]))
+                                        '{0}={1}'.format(key, kwargs[key]))
         else:
             __salt__['file.append'](config_file,
                                     '{0}={1}'.format(key, kwargs[key]))
-    return kwargs
+    return ret_kwargs
 
 
 def get_config(config_file='/etc/dnsmasq.conf'):
@@ -148,6 +157,12 @@ def _parse_dnamasq(filename):
     Generic function for parsing dnsmasq files including includes.
     '''
     fileopts = {}
+
+    if not os.path.isfile(filename):
+        raise CommandExecutionError(
+            'Error: No such file \'{0}\''.format(filename)
+        )
+
     with salt.utils.fopen(filename, 'r') as fp_:
         for line in fp_:
             if not line.strip():

salt/modules/win_file.py

@@ -55,7 +55,7 @@
 from salt.modules.file import (check_hash,  # pylint: disable=W0611
         directory_exists, get_managed, mkdir, makedirs_, makedirs_perms,
         check_managed, check_managed_changes, check_perms, source_list,
-        touch, append, contains, contains_regex,
+        touch, append, contains, contains_regex, get_source_sum,
         contains_glob, find, psed, get_sum, _get_bkroot, _mkstemp_copy,
         get_hash, manage_file, file_exists, get_diff, line, list_backups,
         __clean_tmp, check_file_meta, _binary_replace, restore_backup,
@@ -83,7 +83,7 @@ def __virtual__():
             global source_list, mkdir, __clean_tmp, makedirs_, file_exists
             global check_managed, check_managed_changes, check_file_meta
             global append, _error, directory_exists, touch, contains
-            global contains_regex, contains_glob
+            global contains_regex, contains_glob, get_source_sum
             global find, psed, get_sum, check_hash, get_hash, delete_backup
             global get_diff, _get_flags, extract_hash, comment_line
             global access, copy, readdir, rmdir, truncate, replace, search
@@ -120,6 +120,7 @@ def __virtual__():
             contains = _namespaced_function(contains, globals())
             contains_regex = _namespaced_function(contains_regex, globals())
             contains_glob = _namespaced_function(contains_glob, globals())
+            get_source_sum = _namespaced_function(get_source_sum, globals())
             find = _namespaced_function(find, globals())
             psed = _namespaced_function(psed, globals())
             get_sum = _namespaced_function(get_sum, globals())