If you want to create a dedicated hardware cluster for stateful services in vSphere IaaS control plane, you can use a vSAN Direct datastore. vSAN Direct is a mostly raw datastore that you deploy on unclaimed storage devices local to your ESXi host.
Tag Storage Devices for vSAN Direct
vSAN Direct needs some unclaimed disks on each ESXi host within a vSAN cluster. However, in certain environments, vSAN automatically claims all local storage devices on your hosts. You can make the devices ineligible for regular vSAN and available for vSAN Direct.
Procedure
Use Script to Tag Storage Devices for vSAN Direct
Alternately, you can use the following script to tag HDD devices attached to your ESXi host. After you run the script, the devices become ineligible for regular vSAN and are available for vSAN Direct.
#!/usr/bin/env python3
# Copyright 2020 VMware, Inc. All rights reserved.
# Abstract
#
# This script helps manage tagging of Direct Attached HDD disks
# on ESXi systems for vSAN Direct in preparation for a VCF deployment.
#
# It is expected to be used with ESX systems of version 7.0.1 or later.
#
import argparse
from enum import Enum
import logging
import sys
import os
import paramiko
import subprocess
import traceback
import ast
import getpass
from six.moves import input
from distutils.util import strtobool
from argparse import ArgumentParser
class ParseState(Enum):
OPEN = 0
DEVICE = 1
class RemoteOperationError(Exception):
pass
class EsxVersion:
def __init__(self, major, minor, release):
self.major = major
self.minor = minor
self.release = release
def __str__(self):
return '{}.{}.{}'.format(self.major, self.minor, self.release)
@staticmethod
def build(str):
tokens = str.split(b'.',3)
return EsxVersion(int(tokens[0]), int(tokens[1]), int(tokens[2]))
class StorageDevice:
def __init__(self, deviceId, isSSD, isVsanDirectEnabled):
self.deviceId = str(deviceId.decode())
self.isSSD = isSSD
self.isVsanDirectCapable = True
self.isVsanDirectEnabled = isVsanDirectEnabled
def __str__(self):
return '{}:\n\tIs SSD: {}\n\tvsanDirect enabled:{}'.format(
self.deviceId,
self.isSSD,
self.isVsanDirectEnabled)
@staticmethod
def strToBool(v):
return bool(strtobool(str(v.decode())))
@staticmethod
def build(deviceId, props):
vsanDirectEnabled = False
isLocal = StorageDevice.strToBool(props[b'Is Local'])
status = props[b'Status']
isOffline = StorageDevice.strToBool(props[b'Is Offline'])
isSSD = StorageDevice.strToBool(props[b'Is SSD'])
isBootDevice = StorageDevice.strToBool(props[b'Is Boot Device'])
deviceType = props[b'Device Type']
if deviceType == b'Direct-Access' and isLocal and (not isOffline) and (not isBootDevice) and status == b'on':
return StorageDevice(deviceId, isSSD, vsanDirectEnabled)
else:
print("Skipping device {}".format(deviceId))
return None
def parse_arguments():
"""
Parses the command line arguments to the function
"""
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', dest='hostname',
help='specify hostname for the ESX Server', required=True)
parser.add_argument('--username', dest='username',
help='specify username to connect to the ESX Server', required=True)
parser.add_argument('--password', dest='password',
help='specify password to connect to the ESX Server', required=False)
return parser.parse_args()
def get_esx_version(sshClient):
global logger
stdin_, stdout_, stderr_ = sshClient.exec_command('vmware -v')
exit_status = stdout_.channel.recv_exit_status()
if exit_status != 0:
logger.error('Command exited with non-zero status: %s' % exit_status)
logger.error('Error message: %s' % stderr_.read())
raise RemoteOperationError('Failed to determine ESX version')
output = stdout_.read()
tokens = output.split()
if len(tokens) < 3:
raise RemoteOperationError('Invalid ESX Version - %s', output)
return EsxVersion.build(tokens[2])
def check_esx_version(esxVersion):
return esxVersion.major >= 7 and esxVersion.minor >= 0 and esxVersion.release >= 1
def query_devices(sshClient):
global logger
stdin_, stdout_, stderr_ = sshClient.exec_command('esxcli storage core device list')
exit_status = stdout_.channel.recv_exit_status()
if exit_status != 0:
logger.error('Command exited with non-zero status: %s' % exit_status)
logger.error('Error message: %s' % stderr_.read())
raise RemoteOperationError('Failed to query core storage device list')
output = stdout_.read()
# Build the device list from the output
return create_device_list(output)
def create_device_list(str):
devices = []
deviceId=""
deviceProps={}
parseState = ParseState.OPEN
for line in str.splitlines():
if parseState == ParseState.OPEN:
if line.strip():
deviceId=line.strip()
parseState = ParseState.DEVICE
elif parseState == ParseState.DEVICE:
if line.strip():
props = line.strip().split(b':',1)
deviceProps[props[0]] = props[1].strip()
else:
if deviceId:
device = StorageDevice.build(deviceId, deviceProps)
if device:
devices.append(device)
else:
logger.debug("Skipping device {}".format(deviceId))
deviceId=""
deviceProps={}
parseState = ParseState.OPEN
if deviceId:
device = StorageDevice.build(deviceId, deviceProps)
if device:
devices.append(device)
return devices
def tag_device_for_vsan_direct(sshClient, deviceId):
global logger
logger.info("Tagging device [{}] for vSAN Direct".format(deviceId))
command = "esxcli vsan storage tag add -d " + deviceId + " -t vsanDirect"
stdin_, stdout_, stderr_ = sshClient.exec_command(command)
exit_status = stdout_.channel.recv_exit_status()
if exit_status != 0:
logger.error('Command exited with non-zero status: %s' % exit_status)
logger.error('Error message: %s' % stderr_.read())
raise RemoteOperationError('Failed to tag device [{}] for vSAN Direct'.format(deviceId))
logger.info('Successfully tagged device [{}] for vSAN Direct'.format(deviceId))
def untag_device_for_vsan_direct(sshClient, deviceId):
global logger
logger.info("Untagging device [{}] for vSAN Direct".format(deviceId))
command = "esxcli vsan storage tag remove -d " + deviceId + " -t vsanDirect"
stdin_, stdout_, stderr_ = sshClient.exec_command(command)
exit_status = stdout_.channel.recv_exit_status()
if exit_status != 0:
logger.error('Command exited with non-zero status: %s' % exit_status)
logger.error('Error message: %s' % stderr_.read())
raise RemoteOperationError('Failed to untag device [{}] for vSAN Direct'.format(deviceId))
logger.info('Successfully untagged device [{}] for vSAN Direct'.format(deviceId))
def get_vsan_info_for_device(sshClient, deviceId):
global logger
command = "vdq -q -d {}".format(deviceId)
stdin_, stdout_, stderr_ = sshClient.exec_command(command)
exit_status = stdout_.channel.recv_exit_status()
if exit_status != 0:
logger.error('Command exited with non-zero status: %s' % exit_status)
logger.error('Error message: %s' % stderr_.read())
raise RemoteOperationError('Failed to query vsan direct status on device [%s]' % deviceId)
output = stdout_.read()
return ast.literal_eval(str(output.decode()))
def update_vsan_direct_status(sshClient, devices):
for device in devices:
vsanInfo = get_vsan_info_for_device(sshClient, device.deviceId)
device.isVsanDirectEnabled = vsanInfo[0]['IsVsanDirectDisk'].strip() == "1"
device.isVsanDirectCapable = vsanInfo[0]['State'].strip() == 'Eligible for use by VSAN'
def getVsanDirectCapableDevices(devices):
selectDevices = []
# Cull devices incapable of vSAN Direct
for device in devices:
if device.isVsanDirectCapable:
selectDevices.append(device)
return selectDevices
def print_devices(devices):
print("Direct-Attach Devices:")
print("======================")
iDevice = 0
for device in devices:
iDevice = iDevice + 1
print ("{}. {}".format(iDevice, device))
print("======================")
def tag_devices(sshClient, devices):
for device in devices:
tag_device_for_vsan_direct(sshClient, device.deviceId)
def untag_devices(sshClient, devices):
for device in devices:
untag_device_for_vsan_direct(sshClient, device.deviceId)
def tag_all_hdd_devices(sshClient, devices):
hddDevices = []
for device in devices:
if not device.isSSD:
hddDevices.append(device)
if len(hddDevices) > 0:
tag_devices(sshClient, hddDevices)
def show_usage():
print ("===================================")
print ("commands: {tag-all-hdd, tag, untag}")
print ("\ttag <comma separated serial numbers of devices>")
print ("\tuntag <comma separated serial numbers of devices>")
print ("\ttag-all-hdd")
print ("=============")
def main():
global logger
logger.info('Tag disks for vSAN Direct')
try:
# Parse arguments
args = parse_arguments()
# 1. Setup SSH connection to ESX system
sshClient = paramiko.SSHClient()
sshClient.load_system_host_keys()
sshClient.set_missing_host_key_policy(paramiko.AutoAddPolicy())
passwd = args.password
if passwd == None:
passwd = getpass.getpass(prompt='Password: ')
logger.info('Connecting to ESX System (IP: %s)' % args.hostname)
sshClient.connect(args.hostname, username=args.username, password=passwd)
# version check
esxVersion = get_esx_version(sshClient)
print('ESX Version on {} is {}'.format(args.hostname, esxVersion))
logger.info('Checking ESX Version...')
if not check_esx_version(esxVersion):
raise Exception('ESX Version must be 7.0.1 or greater')
print ('This script helps tag direct-attached disks for vSAN Direct on ESX')
print ('Note: Only disks of type HDD are supported at this time.')
print ()
print ("For help, type help")
show_usage()
while True:
# get device list
print("Querying devices...")
devices = query_devices(sshClient)
# update devices with vSAN Direct status
update_vsan_direct_status(sshClient, devices)
# cull device list
selectDevices = getVsanDirectCapableDevices(devices)
# List the devices for the user to see
print_devices(selectDevices)
# find out what the user wants to do to these devices
args = input('Command> ').split()
if len(args) == 0:
break
cmd = args[0]
if cmd == 'q' or cmd == 'quit' or cmd == 'exit':
break
elif cmd == 'help':
show_usage()
elif cmd == 'tag-all-hdd':
print("Tagging all HDD devices...")
tag_all_hdd_devices(sshClient, selectDevices)
elif cmd == 'tag' or cmd == 'untag':
chosenDevices = []
if len(args) > 1:
serials = args[1].split(',')
for serialStr in serials:
serial = int(serialStr)
if serial < 1 or serial > len(selectDevices):
raise Exception("Error: Serial {} is out of range".format(serial))
chosenDevices.append(selectDevices[serial-1])
if len(chosenDevices) == 0:
print("No devices specified")
continue
if cmd == 'tag':
print("Tagging devices...")
tag_devices(sshClient, chosenDevices)
else:
print("Untagging devices...")
untag_devices(sshClient, chosenDevices)
else:
print ("Error: Unrecognized command - %s" % cmd)
except paramiko.ssh_exception.AuthenticationException as e:
logger.error(e)
sys.exit(5)
except Exception as e:
logger.error('Disk tagging failed with error: %s' % e)
logger.error(traceback.format_exc())
sys.exit(1)
finally:
# Close SSH client
try:
sshClient.close()
except:
pass
# Set up logging
logging.basicConfig()
logger = logging.getLogger('tag-disks-for-vsan-direct')
if __name__ == "__main__":
main()
Create a vSAN Direct Datastore
As a vSphere administrator, set up a vSAN Direct datastore to be used with such functionality as vSAN Data Persistence platform or VM instance storage. To create the datastore, use unclaimed storage devices local to your ESXi host.
Procedure
- Select a device to claim and select a check box in the Claim for vSAN Direct column.
Note: If you claimed the devices for a regular vSAN datastore, these devices do not appear in the vSAN Direct tab.
- Click the Datastores tab to display all vSAN Direct datastores in your cluster.
