|
|
|
|
@@ -33,6 +33,7 @@ except ImportError:
|
|
|
|
|
import random
|
|
|
|
|
import re
|
|
|
|
|
import requests
|
|
|
|
|
import socket
|
|
|
|
|
import sys
|
|
|
|
|
import time
|
|
|
|
|
import urllib3
|
|
|
|
|
@@ -40,7 +41,7 @@ import urllib3
|
|
|
|
|
|
|
|
|
|
# Constants
|
|
|
|
|
__appname__ = "ProxLB"
|
|
|
|
|
__version__ = "1.0.0"
|
|
|
|
|
__version__ = "1.0.3"
|
|
|
|
|
__author__ = "Florian Paul Azim Hoberg <gyptazy@gyptazy.ch> @gyptazy"
|
|
|
|
|
__errors__ = False
|
|
|
|
|
|
|
|
|
|
@@ -112,7 +113,7 @@ def validate_daemon(daemon, schedule):
|
|
|
|
|
|
|
|
|
|
if bool(int(daemon)):
|
|
|
|
|
logging.info(f'{info_prefix} Running in daemon mode. Next run in {schedule} hours.')
|
|
|
|
|
time.sleep(int(schedule) * 60)
|
|
|
|
|
time.sleep(int(schedule) * 60 * 60)
|
|
|
|
|
else:
|
|
|
|
|
logging.info(f'{info_prefix} Not running in daemon mode. Quitting.')
|
|
|
|
|
sys.exit(0)
|
|
|
|
|
@@ -145,7 +146,7 @@ def __validate_config_file(config_path):
|
|
|
|
|
def initialize_args():
|
|
|
|
|
""" Initialize given arguments for ProxLB. """
|
|
|
|
|
argparser = argparse.ArgumentParser(description='ProxLB')
|
|
|
|
|
argparser.add_argument('-c', '--config', type=str, help='Path to config file.', required=True)
|
|
|
|
|
argparser.add_argument('-c', '--config', type=str, help='Path to config file.', required=False)
|
|
|
|
|
argparser.add_argument('-d', '--dry-run', help='Perform a dry-run without doing any actions.', action='store_true', required=False)
|
|
|
|
|
argparser.add_argument('-j', '--json', help='Return a JSON of the VM movement.', action='store_true', required=False)
|
|
|
|
|
return argparser.parse_args()
|
|
|
|
|
@@ -173,22 +174,30 @@ def initialize_config_options(config_path):
|
|
|
|
|
config = configparser.ConfigParser()
|
|
|
|
|
config.read(config_path)
|
|
|
|
|
# Proxmox config
|
|
|
|
|
proxmox_api_host = config['proxmox']['api_host']
|
|
|
|
|
proxmox_api_user = config['proxmox']['api_user']
|
|
|
|
|
proxmox_api_pass = config['proxmox']['api_pass']
|
|
|
|
|
proxmox_api_ssl_v = config['proxmox']['verify_ssl']
|
|
|
|
|
# Balancing
|
|
|
|
|
balancing_method = config['balancing'].get('method', 'memory')
|
|
|
|
|
balancing_mode = config['balancing'].get('mode', 'used')
|
|
|
|
|
balancing_mode_option = config['balancing'].get('mode_option', 'bytes')
|
|
|
|
|
balancing_type = config['balancing'].get('type', 'vm')
|
|
|
|
|
balanciness = config['balancing'].get('balanciness', 10)
|
|
|
|
|
ignore_nodes = config['balancing'].get('ignore_nodes', None)
|
|
|
|
|
ignore_vms = config['balancing'].get('ignore_vms', None)
|
|
|
|
|
proxmox_api_host = config['proxmox']['api_host']
|
|
|
|
|
proxmox_api_user = config['proxmox']['api_user']
|
|
|
|
|
proxmox_api_pass = config['proxmox']['api_pass']
|
|
|
|
|
proxmox_api_ssl_v = config['proxmox']['verify_ssl']
|
|
|
|
|
# VM Balancing
|
|
|
|
|
vm_balancing_enable = config['vm_balancing'].get('enable', 1)
|
|
|
|
|
vm_balancing_method = config['vm_balancing'].get('method', 'memory')
|
|
|
|
|
vm_balancing_mode = config['vm_balancing'].get('mode', 'used')
|
|
|
|
|
vm_balancing_mode_option = config['vm_balancing'].get('mode_option', 'bytes')
|
|
|
|
|
vm_balancing_type = config['vm_balancing'].get('type', 'vm')
|
|
|
|
|
vm_balanciness = config['vm_balancing'].get('balanciness', 10)
|
|
|
|
|
vm_parallel_migrations = config['vm_balancing'].get('parallel_migrations', 1)
|
|
|
|
|
vm_ignore_nodes = config['vm_balancing'].get('ignore_nodes', None)
|
|
|
|
|
vm_ignore_vms = config['vm_balancing'].get('ignore_vms', None)
|
|
|
|
|
# Storage Balancing
|
|
|
|
|
storage_balancing_enable = config['storage_balancing'].get('enable', 0)
|
|
|
|
|
storage_balancing_method = config['storage_balancing'].get('method', 'disk_space')
|
|
|
|
|
storage_balancing_balanciness = config['storage_balancing'].get('balanciness', 10)
|
|
|
|
|
storage_parallel_migrations = config['storage_balancing'].get('parallel_migrations', 0)
|
|
|
|
|
# Service
|
|
|
|
|
daemon = config['service'].get('daemon', 1)
|
|
|
|
|
schedule = config['service'].get('schedule', 24)
|
|
|
|
|
log_verbosity = config['service'].get('log_verbosity', 'CRITICAL')
|
|
|
|
|
master_only = config['service'].get('master_only', 0)
|
|
|
|
|
daemon = config['service'].get('daemon', 1)
|
|
|
|
|
schedule = config['service'].get('schedule', 24)
|
|
|
|
|
log_verbosity = config['service'].get('log_verbosity', 'CRITICAL')
|
|
|
|
|
except configparser.NoSectionError:
|
|
|
|
|
logging.critical(f'{error_prefix} Could not find the required section.')
|
|
|
|
|
sys.exit(2)
|
|
|
|
|
@@ -200,8 +209,8 @@ def initialize_config_options(config_path):
|
|
|
|
|
sys.exit(2)
|
|
|
|
|
|
|
|
|
|
logging.info(f'{info_prefix} Configuration file loaded.')
|
|
|
|
|
return proxmox_api_host, proxmox_api_user, proxmox_api_pass, proxmox_api_ssl_v, balancing_method, balancing_mode, \
|
|
|
|
|
balancing_mode_option, balancing_type, balanciness, ignore_nodes, ignore_vms, daemon, schedule, log_verbosity
|
|
|
|
|
return proxmox_api_host, proxmox_api_user, proxmox_api_pass, proxmox_api_ssl_v, vm_balancing_enable, vm_balancing_method, vm_balancing_mode, vm_balancing_mode_option, \
|
|
|
|
|
vm_balancing_type, vm_balanciness, vm_parallel_migrations, vm_ignore_nodes, vm_ignore_vms, storage_balancing_enable, storage_balancing_method, storage_balancing_balanciness, storage_parallel_migrations, master_only, daemon, schedule, log_verbosity
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def api_connect(proxmox_api_host, proxmox_api_user, proxmox_api_pass, proxmox_api_ssl_v):
|
|
|
|
|
@@ -231,6 +240,62 @@ def api_connect(proxmox_api_host, proxmox_api_user, proxmox_api_pass, proxmox_ap
|
|
|
|
|
return api_object
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def execute_rebalancing_only_by_master(api_object, master_only):
|
|
|
|
|
""" Validate if balancing should only be done by the cluster master. Afterwards, validate if this node is the cluster master. """
|
|
|
|
|
info_prefix = 'Info: [only-on-master-executor]:'
|
|
|
|
|
master_only = bool(int(master_only))
|
|
|
|
|
|
|
|
|
|
if bool(int(master_only)):
|
|
|
|
|
logging.info(f'{info_prefix} Master only rebalancing is defined. Starting validation.')
|
|
|
|
|
cluster_master_node = get_cluster_master(api_object)
|
|
|
|
|
cluster_master = validate_cluster_master(cluster_master_node)
|
|
|
|
|
return cluster_master, master_only
|
|
|
|
|
else:
|
|
|
|
|
logging.info(f'{info_prefix} No master only rebalancing is defined. Skipping validation.')
|
|
|
|
|
return False, master_only
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_cluster_master(api_object):
|
|
|
|
|
""" Get the current master of the Proxmox cluster. """
|
|
|
|
|
error_prefix = 'Error: [cluster-master-getter]:'
|
|
|
|
|
info_prefix = 'Info: [cluster-master-getter]:'
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
ha_status_object = api_object.cluster().ha().status().manager_status().get()
|
|
|
|
|
logging.info(f'{info_prefix} Master node: {ha_status_object.get("manager_status", None).get("master_node", None)}')
|
|
|
|
|
except urllib3.exceptions.NameResolutionError:
|
|
|
|
|
logging.critical(f'{error_prefix} Could not resolve the API.')
|
|
|
|
|
sys.exit(2)
|
|
|
|
|
except requests.exceptions.ConnectTimeout:
|
|
|
|
|
logging.critical(f'{error_prefix} Connection time out to API.')
|
|
|
|
|
sys.exit(2)
|
|
|
|
|
except requests.exceptions.SSLError:
|
|
|
|
|
logging.critical(f'{error_prefix} SSL certificate verification failed for API.')
|
|
|
|
|
sys.exit(2)
|
|
|
|
|
|
|
|
|
|
cluster_master = ha_status_object.get("manager_status", None).get("master_node", None)
|
|
|
|
|
|
|
|
|
|
if cluster_master:
|
|
|
|
|
return cluster_master
|
|
|
|
|
else:
|
|
|
|
|
logging.critical(f'{error_prefix} Could not obtain cluster master. Please check your configuration - stopping.')
|
|
|
|
|
sys.exit(2)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def validate_cluster_master(cluster_master):
|
|
|
|
|
""" Validate if the current execution node is the cluster master. """
|
|
|
|
|
info_prefix = 'Info: [cluster-master-validator]:'
|
|
|
|
|
|
|
|
|
|
node_executor_hostname = socket.gethostname()
|
|
|
|
|
logging.info(f'{info_prefix} Node executor hostname is: {node_executor_hostname}')
|
|
|
|
|
|
|
|
|
|
if node_executor_hostname != cluster_master:
|
|
|
|
|
logging.info(f'{info_prefix} {node_executor_hostname} is not the cluster master ({cluster_master}).')
|
|
|
|
|
return False
|
|
|
|
|
else:
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_node_statistics(api_object, ignore_nodes):
|
|
|
|
|
""" Get statistics of cpu, memory and disk for each node in the cluster. """
|
|
|
|
|
info_prefix = 'Info: [node-statistics]:'
|
|
|
|
|
@@ -316,7 +381,37 @@ def get_vm_statistics(api_object, ignore_vms, balancing_type):
|
|
|
|
|
vm_statistics[vm['name']]['disk_used'] = vm['disk']
|
|
|
|
|
vm_statistics[vm['name']]['vmid'] = vm['vmid']
|
|
|
|
|
vm_statistics[vm['name']]['node_parent'] = node['node']
|
|
|
|
|
vm_statistics[vm['name']]['storages'] = {}
|
|
|
|
|
vm_statistics[vm['name']]['type'] = 'vm'
|
|
|
|
|
|
|
|
|
|
# Get disk details of the related object.
|
|
|
|
|
_vm_details = api_object.nodes(node['node']).qemu(vm['vmid']).config.get()
|
|
|
|
|
_vm_details_storage_allowed = ['ide', 'nvme', 'scsi', 'virtio', 'sata']
|
|
|
|
|
logging.info(f'{info_prefix} Getting disk information for vm {vm["name"]}.')
|
|
|
|
|
|
|
|
|
|
for vm_detail_key, vm_detail_value in _vm_details.items():
|
|
|
|
|
vm_detail_key_validator = re.sub("\d+$", "", vm_detail_key)
|
|
|
|
|
|
|
|
|
|
if vm_detail_key_validator in _vm_details_storage_allowed:
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key] = {}
|
|
|
|
|
match = re.match(r'([^:]+):[^/]+/(.+),iothread=\d+,size=(\d+G)', _vm_details[vm_detail_key])
|
|
|
|
|
|
|
|
|
|
# Create an efficient match group and split the strings to assign them to the storage information.
|
|
|
|
|
if match:
|
|
|
|
|
_volume = match.group(1)
|
|
|
|
|
_disk_name = match.group(2)
|
|
|
|
|
_disk_size = match.group(3)
|
|
|
|
|
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key]['name'] = _disk_name
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key]['device_name'] = vm_detail_key
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key]['volume'] = _volume
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key]['storage_parent'] = _volume
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key]['storage_rebalance'] = None
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key]['size'] = _disk_size
|
|
|
|
|
logging.info(f'{info_prefix} Added disk for {vm["name"]}: Name {_disk_name} on volume {_volume} with size {_disk_size}.')
|
|
|
|
|
else:
|
|
|
|
|
logging.info(f'{info_prefix} No disks for {vm["name"]} found.')
|
|
|
|
|
|
|
|
|
|
# Rebalancing node will be overwritten after calculations.
|
|
|
|
|
# If the vm stays on the node, it will be removed at a
|
|
|
|
|
# later time.
|
|
|
|
|
@@ -354,7 +449,37 @@ def get_vm_statistics(api_object, ignore_vms, balancing_type):
|
|
|
|
|
vm_statistics[vm['name']]['disk_used'] = vm['disk']
|
|
|
|
|
vm_statistics[vm['name']]['vmid'] = vm['vmid']
|
|
|
|
|
vm_statistics[vm['name']]['node_parent'] = node['node']
|
|
|
|
|
vm_statistics[vm['name']]['storages'] = {}
|
|
|
|
|
vm_statistics[vm['name']]['type'] = 'ct'
|
|
|
|
|
|
|
|
|
|
# Get disk details of the related object.
|
|
|
|
|
_vm_details = api_object.nodes(node['node']).qemu(vm['vmid']).config.get()
|
|
|
|
|
_vm_details_storage_allowed = ['ide', 'nvme', 'scsi']
|
|
|
|
|
logging.info(f'{info_prefix} Getting disk information for vm {vm["name"]}.')
|
|
|
|
|
|
|
|
|
|
for vm_detail_key, vm_detail_value in _vm_details.items():
|
|
|
|
|
vm_detail_key_validator = re.sub("\d+$", "", vm_detail_key)
|
|
|
|
|
|
|
|
|
|
if vm_detail_key_validator in _vm_details_storage_allowed:
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key] = {}
|
|
|
|
|
match = re.match(r'([^:]+):[^/]+/(.+),iothread=\d+,size=(\d+G)', _vm_details[vm_detail_key])
|
|
|
|
|
|
|
|
|
|
# Create an efficient match group and split the strings to assign them to the storage information.
|
|
|
|
|
if match:
|
|
|
|
|
_volume = match.group(1)
|
|
|
|
|
_disk_name = match.group(2)
|
|
|
|
|
_disk_size = match.group(3)
|
|
|
|
|
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key]['name'] = _disk_name
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key]['device_name'] = vm_detail_key
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key]['volume'] = _volume
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key]['storage_parent'] = _volume
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key]['storage_rebalance'] = None
|
|
|
|
|
vm_statistics[vm['name']]['storages'][vm_detail_key]['size'] = _disk_size
|
|
|
|
|
logging.info(f'{info_prefix} Added disk for {vm["name"]}: Name {_disk_name} on volume {_volume} with size {_disk_size}.')
|
|
|
|
|
else:
|
|
|
|
|
logging.info(f'{info_prefix} No disks for {vm["name"]} found.')
|
|
|
|
|
|
|
|
|
|
# Rebalancing node will be overwritten after calculations.
|
|
|
|
|
# If the vm stays on the node, it will be removed at a
|
|
|
|
|
# later time.
|
|
|
|
|
@@ -365,6 +490,58 @@ def get_vm_statistics(api_object, ignore_vms, balancing_type):
|
|
|
|
|
return vm_statistics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_storage_statistics(api_object):
|
|
|
|
|
""" Get statistics of all storages in the cluster. """
|
|
|
|
|
info_prefix = 'Info: [storage-statistics]:'
|
|
|
|
|
warn_prefix = 'Warn: [storage-statistics]:'
|
|
|
|
|
storage_statistics = {}
|
|
|
|
|
|
|
|
|
|
for node in api_object.nodes.get():
|
|
|
|
|
|
|
|
|
|
for storage in api_object.nodes(node['node']).storage.get():
|
|
|
|
|
|
|
|
|
|
# Only add enabled and active storage repositories that might be suitable for further
|
|
|
|
|
# storage balancing.
|
|
|
|
|
if storage['enabled'] and storage['active'] and storage['shared']:
|
|
|
|
|
storage_statistics[storage['storage']] = {}
|
|
|
|
|
storage_statistics[storage['storage']]['name'] = storage['storage']
|
|
|
|
|
storage_statistics[storage['storage']]['total'] = storage['total']
|
|
|
|
|
storage_statistics[storage['storage']]['used'] = storage['used']
|
|
|
|
|
storage_statistics[storage['storage']]['used_percent'] = storage['used'] / storage['total'] * 100
|
|
|
|
|
storage_statistics[storage['storage']]['used_percent_last_run'] = 0
|
|
|
|
|
storage_statistics[storage['storage']]['free'] = storage['total'] - storage['used']
|
|
|
|
|
storage_statistics[storage['storage']]['free_percent'] = storage_statistics[storage['storage']]['free'] / storage['total'] * 100
|
|
|
|
|
storage_statistics[storage['storage']]['used_fraction'] = storage['used_fraction']
|
|
|
|
|
storage_statistics[storage['storage']]['type'] = storage['type']
|
|
|
|
|
storage_statistics[storage['storage']]['content'] = storage['content']
|
|
|
|
|
storage_statistics[storage['storage']]['usage_type'] = ''
|
|
|
|
|
|
|
|
|
|
# Split the Proxmox returned values to a list and validate the supported
|
|
|
|
|
# types of the underlying storage for further migrations.
|
|
|
|
|
storage_content_list = storage['content'].split(',')
|
|
|
|
|
usage_ct = False
|
|
|
|
|
usage_vm = False
|
|
|
|
|
|
|
|
|
|
if 'rootdir' in storage_content_list:
|
|
|
|
|
usage_ct = True
|
|
|
|
|
storage_statistics[storage['storage']]['usage_type'] = 'ct'
|
|
|
|
|
logging.info(f'{info_prefix} Storage {storage["storage"]} support CTs.')
|
|
|
|
|
|
|
|
|
|
if 'images' in storage_content_list:
|
|
|
|
|
usage_vm = True
|
|
|
|
|
storage_statistics[storage['storage']]['usage_type'] = 'vm'
|
|
|
|
|
logging.info(f'{info_prefix} Storage {storage["storage"]} support VMs.')
|
|
|
|
|
|
|
|
|
|
if usage_ct and usage_vm:
|
|
|
|
|
storage_statistics[storage['storage']]['usage_type'] = 'all'
|
|
|
|
|
logging.info(f'{info_prefix} Updateing storage {storage["storage"]} support to CTs and VMs.')
|
|
|
|
|
|
|
|
|
|
logging.info(f'{info_prefix} Added storage {storage["storage"]}.')
|
|
|
|
|
|
|
|
|
|
logging.info(f'{info_prefix} Created storage statistics.')
|
|
|
|
|
return storage_statistics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def update_node_statistics(node_statistics, vm_statistics):
|
|
|
|
|
""" Update node statistics by VMs statistics. """
|
|
|
|
|
info_prefix = 'Info: [node-update-statistics]:'
|
|
|
|
|
@@ -445,15 +622,15 @@ def __get_proxlb_groups(vm_tags):
|
|
|
|
|
return group_include, group_exclude, vm_ignore
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def balancing_calculations(balancing_method, balancing_mode, balancing_mode_option, node_statistics, vm_statistics, balanciness, rebalance, processed_vms):
|
|
|
|
|
def balancing_vm_calculations(balancing_method, balancing_mode, balancing_mode_option, node_statistics, vm_statistics, balanciness, rebalance, processed_vms):
|
|
|
|
|
""" Calculate re-balancing of VMs on present nodes across the cluster. """
|
|
|
|
|
info_prefix = 'Info: [rebalancing-calculator]:'
|
|
|
|
|
info_prefix = 'Info: [vm-rebalancing-calculator]:'
|
|
|
|
|
|
|
|
|
|
# Validate for a supported balancing method, mode and if rebalancing is required.
|
|
|
|
|
__validate_balancing_method(balancing_method)
|
|
|
|
|
__validate_balancing_mode(balancing_mode)
|
|
|
|
|
__validate_vm_statistics(vm_statistics)
|
|
|
|
|
rebalance = __validate_balanciness(balanciness, balancing_method, balancing_mode, node_statistics)
|
|
|
|
|
rebalance = __validate_vm_balanciness(balanciness, balancing_method, balancing_mode, node_statistics)
|
|
|
|
|
|
|
|
|
|
if rebalance:
|
|
|
|
|
# Get most used/assigned resources of the VM and the most free or less allocated node.
|
|
|
|
|
@@ -465,7 +642,7 @@ def balancing_calculations(balancing_method, balancing_mode, balancing_mode_opti
|
|
|
|
|
vm_statistics, node_statistics, balancing_method, balancing_mode)
|
|
|
|
|
|
|
|
|
|
# Start recursion until we do not have any needs to rebalance anymore.
|
|
|
|
|
balancing_calculations(balancing_method, balancing_mode, balancing_mode_option, node_statistics, vm_statistics, balanciness, rebalance, processed_vms)
|
|
|
|
|
balancing_vm_calculations(balancing_method, balancing_mode, balancing_mode_option, node_statistics, vm_statistics, balanciness, rebalance, processed_vms)
|
|
|
|
|
|
|
|
|
|
# Honour groupings for include and exclude groups for rebalancing VMs.
|
|
|
|
|
node_statistics, vm_statistics = __get_vm_tags_include_groups(vm_statistics, node_statistics, balancing_method, balancing_mode)
|
|
|
|
|
@@ -480,10 +657,47 @@ def balancing_calculations(balancing_method, balancing_mode, balancing_mode_opti
|
|
|
|
|
return node_statistics, vm_statistics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def balancing_storage_calculations(storage_balancing_method, storage_statistics, vm_statistics, balanciness, rebalance, processed_vms):
|
|
|
|
|
""" Calculate re-balancing of storage on present datastores across the cluster. """
|
|
|
|
|
info_prefix = 'Info: [storage-rebalancing-calculator]:'
|
|
|
|
|
|
|
|
|
|
# Validate for a supported balancing method, mode and if rebalancing is required.
|
|
|
|
|
__validate_vm_statistics(vm_statistics)
|
|
|
|
|
|
|
|
|
|
rebalance = __validate_storage_balanciness(balanciness, storage_balancing_method, storage_statistics)
|
|
|
|
|
|
|
|
|
|
if rebalance:
|
|
|
|
|
vm_name, vm_disk_device, vm_disk_name = __get_most_used_resources_vm_storage(vm_statistics)
|
|
|
|
|
if vm_name not in processed_vms:
|
|
|
|
|
processed_vms.append(vm_name)
|
|
|
|
|
resources_storage_most_free = __get_most_free_storage(storage_balancing_method, storage_statistics)
|
|
|
|
|
|
|
|
|
|
# Update resource statistics for VMs and storage.
|
|
|
|
|
storage_statistics, vm_statistics = __update_resource_storage_statistics(storage_statistics, resources_storage_most_free, vm_statistics, vm_name, vm_disk_device)
|
|
|
|
|
|
|
|
|
|
# Start recursion until we do not have any needs to rebalance anymore.
|
|
|
|
|
balancing_storage_calculations(storage_balancing_method, storage_statistics, vm_statistics, balanciness, rebalance, processed_vms)
|
|
|
|
|
|
|
|
|
|
# Remove VMs where their storage is not being relocated.
|
|
|
|
|
vms_to_remove = [vm_name for vm_name, vm_info in vm_statistics.items() if all(storage.get('storage_rebalance') is None for storage in vm_info.get('storages', {}).values())]
|
|
|
|
|
for vm_name in vms_to_remove:
|
|
|
|
|
del vm_statistics[vm_name]
|
|
|
|
|
|
|
|
|
|
logging.info(f'{info_prefix} Balancing calculations done.')
|
|
|
|
|
return storage_statistics, vm_statistics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def size_in_bytes(size_str):
|
|
|
|
|
size_unit = size_str[-1].upper()
|
|
|
|
|
size_value = float(size_str[:-1])
|
|
|
|
|
size_multipliers = {'K': 1024, 'M': 1024**2, 'G': 1024**3, 'T': 1024**4}
|
|
|
|
|
return size_value * size_multipliers.get(size_unit, 1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __validate_balancing_method(balancing_method):
|
|
|
|
|
""" Validate for valid and supported balancing method. """
|
|
|
|
|
error_prefix = 'Error: [balancing-method-validation]:'
|
|
|
|
|
info_prefix = 'Info: [balancing-method-validation]]:'
|
|
|
|
|
info_prefix = 'Info: [balancing-method-validation]:'
|
|
|
|
|
|
|
|
|
|
if balancing_method not in ['memory', 'disk', 'cpu']:
|
|
|
|
|
logging.error(f'{error_prefix} Invalid balancing method: {balancing_method}')
|
|
|
|
|
@@ -495,9 +709,9 @@ def __validate_balancing_method(balancing_method):
|
|
|
|
|
def __validate_balancing_mode(balancing_mode):
|
|
|
|
|
""" Validate for valid and supported balancing mode. """
|
|
|
|
|
error_prefix = 'Error: [balancing-mode-validation]:'
|
|
|
|
|
info_prefix = 'Info: [balancing-mode-validation]]:'
|
|
|
|
|
info_prefix = 'Info: [balancing-mode-validation]:'
|
|
|
|
|
|
|
|
|
|
if balancing_mode not in ['used', 'assigned']:
|
|
|
|
|
if balancing_mode not in ['used', 'assigned', 'io']:
|
|
|
|
|
logging.error(f'{error_prefix} Invalid balancing method: {balancing_mode}')
|
|
|
|
|
sys.exit(2)
|
|
|
|
|
else:
|
|
|
|
|
@@ -513,10 +727,10 @@ def __validate_vm_statistics(vm_statistics):
|
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __validate_balanciness(balanciness, balancing_method, balancing_mode, node_statistics):
|
|
|
|
|
""" Validate for balanciness to ensure further rebalancing is needed. """
|
|
|
|
|
info_prefix = 'Info: [balanciness-validation]:'
|
|
|
|
|
node_resource_percent_list = []
|
|
|
|
|
def __validate_vm_balanciness(balanciness, balancing_method, balancing_mode, node_statistics):
|
|
|
|
|
""" Validate for balanciness for VMs/CTs to ensure further rebalancing is needed. """
|
|
|
|
|
info_prefix = 'Info: [vm-balanciness-validation]:'
|
|
|
|
|
node_resource_percent_list = []
|
|
|
|
|
node_assigned_percent_match = []
|
|
|
|
|
|
|
|
|
|
# Remap balancing mode to get the related values from nodes dict.
|
|
|
|
|
@@ -532,6 +746,7 @@ def __validate_balanciness(balanciness, balancing_method, balancing_mode, node_s
|
|
|
|
|
node_statistics[node_name][f'{balancing_method}_{node_resource_selector}_percent_match'] = True
|
|
|
|
|
else:
|
|
|
|
|
node_statistics[node_name][f'{balancing_method}_{node_resource_selector}_percent_match'] = False
|
|
|
|
|
|
|
|
|
|
# Update value to the current value of the recursion run.
|
|
|
|
|
node_statistics[node_name][f'{balancing_method}_{node_resource_selector}_percent_last_run'] = node_statistics[node_name][f'{balancing_method}_{node_resource_selector}_percent']
|
|
|
|
|
|
|
|
|
|
@@ -553,10 +768,66 @@ def __validate_balanciness(balanciness, balancing_method, balancing_mode, node_s
|
|
|
|
|
|
|
|
|
|
# Validate if the recursion should be proceeded for further rebalancing.
|
|
|
|
|
if (int(node_lowest_percent) + int(balanciness)) < int(node_highest_percent):
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing for {balancing_method} is needed. Highest usage: {int(node_highest_percent)}% | Lowest usage: {int(node_lowest_percent)}%.')
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing for {balancing_method} of VMs/CTs is needed. Highest usage: {int(node_highest_percent)}% | Lowest usage: {int(node_lowest_percent)}%.')
|
|
|
|
|
return True
|
|
|
|
|
else:
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing for {balancing_method} is not needed. Highest usage: {int(node_highest_percent)}% | Lowest usage: {int(node_lowest_percent)}%.')
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing for {balancing_method} of VMs/CTs is not needed. Highest usage: {int(node_highest_percent)}% | Lowest usage: {int(node_lowest_percent)}%.')
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __validate_storage_balanciness(balanciness, storage_balancing_method, storage_statistics):
|
|
|
|
|
""" Validate for balanciness of storage to ensure further rebalancing is needed. """
|
|
|
|
|
info_prefix = 'Info: [storage-balanciness-validation]:'
|
|
|
|
|
error_prefix = 'Error: [storage-balanciness-validation]:'
|
|
|
|
|
storage_resource_percent_list = []
|
|
|
|
|
storage_assigned_percent_match = []
|
|
|
|
|
|
|
|
|
|
# Validate for an allowed balancing method and define the storage resource selector.
|
|
|
|
|
if storage_balancing_method == 'disk_space':
|
|
|
|
|
logging.info(f'{info_prefix} Getting most free storage volume by disk size..')
|
|
|
|
|
storage_resource_selector = 'used'
|
|
|
|
|
elif storage_balancing_method == 'disk_io':
|
|
|
|
|
logging.error(f'{error_prefix} Getting most free storage volume by disk IO is not yet supported.')
|
|
|
|
|
sys.exit(2)
|
|
|
|
|
else:
|
|
|
|
|
logging.error(f'{error_prefix} Getting most free storage volume by disk IO is not yet supported.')
|
|
|
|
|
sys.exit(2)
|
|
|
|
|
|
|
|
|
|
# Obtain the metrics
|
|
|
|
|
for storage_name, storage_info in storage_statistics.items():
|
|
|
|
|
|
|
|
|
|
logging.info(f'{info_prefix} Validating storage: {storage_name} for balanciness for usage with: {storage_balancing_method}.')
|
|
|
|
|
# Save information of nodes from current run to compare them in the next recursion.
|
|
|
|
|
if storage_statistics[storage_name][f'{storage_resource_selector}_percent_last_run'] == storage_statistics[storage_name][f'{storage_resource_selector}_percent']:
|
|
|
|
|
storage_statistics[storage_name][f'{storage_resource_selector}_percent_match'] = True
|
|
|
|
|
else:
|
|
|
|
|
storage_statistics[storage_name][f'{storage_resource_selector}_percent_match'] = False
|
|
|
|
|
|
|
|
|
|
# Update value to the current value of the recursion run.
|
|
|
|
|
storage_statistics[storage_name][f'{storage_resource_selector}_percent_last_run'] = storage_statistics[storage_name][f'{storage_resource_selector}_percent']
|
|
|
|
|
|
|
|
|
|
# If all node resources are unchanged, the recursion can be left.
|
|
|
|
|
for key, value in storage_statistics.items():
|
|
|
|
|
storage_assigned_percent_match.append(value.get(f'{storage_resource_selector}_percent_match', False))
|
|
|
|
|
|
|
|
|
|
if False not in storage_assigned_percent_match:
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
# Add node information to resource list.
|
|
|
|
|
storage_resource_percent_list.append(int(storage_info[f'{storage_resource_selector}_percent']))
|
|
|
|
|
logging.info(f'{info_prefix} Storage: {storage_name} with values: {storage_info}')
|
|
|
|
|
|
|
|
|
|
# Create a sorted list of the delta + balanciness between the node resources.
|
|
|
|
|
storage_resource_percent_list_sorted = sorted(storage_resource_percent_list)
|
|
|
|
|
storage_lowest_percent = storage_resource_percent_list_sorted[0]
|
|
|
|
|
storage_highest_percent = storage_resource_percent_list_sorted[-1]
|
|
|
|
|
|
|
|
|
|
# Validate if the recursion should be proceeded for further rebalancing.
|
|
|
|
|
if (int(storage_lowest_percent) + int(balanciness)) < int(storage_highest_percent):
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing for type "{storage_resource_selector}" of storage is needed. Highest usage: {int(storage_highest_percent)}% | Lowest usage: {int(storage_lowest_percent)}%.')
|
|
|
|
|
return True
|
|
|
|
|
else:
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing for type "{storage_resource_selector}" of storage is not needed. Highest usage: {int(storage_highest_percent)}% | Lowest usage: {int(storage_lowest_percent)}%.')
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@@ -593,6 +864,68 @@ def __get_most_free_resources_node(balancing_method, balancing_mode, balancing_m
|
|
|
|
|
return node
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __get_most_used_resources_vm_storage(vm_statistics):
|
|
|
|
|
""" Get and return the most used disk of a VM by storage. """
|
|
|
|
|
info_prefix = 'Info: [get-most-used-disks-resources-vm]:'
|
|
|
|
|
biggest_entry = None
|
|
|
|
|
vm_name = None
|
|
|
|
|
max_size = 0
|
|
|
|
|
|
|
|
|
|
# Get the biggest storage of a VM/CT. A VM/CT can hold multiple disks. Therefore, we need to iterate
|
|
|
|
|
# over all assigned disks to get the biggest one.
|
|
|
|
|
vm_biggest_storage = sorted(
|
|
|
|
|
vm_statistics.items(),
|
|
|
|
|
key=lambda x: max(
|
|
|
|
|
(size_in_bytes(storage['size']) for storage in x[1].get('storages', {}).values() if 'size' in storage),
|
|
|
|
|
default=0
|
|
|
|
|
),
|
|
|
|
|
reverse=True
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Iterate over all attached disks to the VM that were returned for the VM object holding the biggest disk.
|
|
|
|
|
# Return: disk name and parent stroage volume for further calculations and migrations
|
|
|
|
|
for key, value in vm_biggest_storage[0][1]['storages'].items():
|
|
|
|
|
if 'size' in value and value['size']:
|
|
|
|
|
size_str = value['size'].upper()
|
|
|
|
|
if size_str.endswith('G'):
|
|
|
|
|
size = float(size_str[:-1]) * 1024 # Convert GB to MB
|
|
|
|
|
elif size_str.endswith('M'):
|
|
|
|
|
size = float(size_str[:-1])
|
|
|
|
|
elif size_str.endswith('T'):
|
|
|
|
|
size = float(size_str[:-1]) * 1024 * 1024 # Convert TB to MB
|
|
|
|
|
else:
|
|
|
|
|
size = 0
|
|
|
|
|
|
|
|
|
|
if size > max_size:
|
|
|
|
|
max_size = size
|
|
|
|
|
vm_name = vm_biggest_storage[0][0]
|
|
|
|
|
biggest_entry = value
|
|
|
|
|
|
|
|
|
|
if biggest_entry:
|
|
|
|
|
return vm_name, biggest_entry['device_name'], biggest_entry['storage_parent']
|
|
|
|
|
else:
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __get_most_free_storage(storage_balancing_method, storage_statistics):
|
|
|
|
|
""" Get the storage with the most free space or IO, depending on the balancing mode. """
|
|
|
|
|
info_prefix = 'Info: [get-most-free-storage]:'
|
|
|
|
|
error_prefix = 'Error: [get-most-free-storage]:'
|
|
|
|
|
storage_volume = None
|
|
|
|
|
logging.info(f'{info_prefix} Starting to evaluate the most free storage volume.')
|
|
|
|
|
|
|
|
|
|
if storage_balancing_method == 'disk_space':
|
|
|
|
|
logging.info(f'{info_prefix} Getting most free storage volume by disk space.')
|
|
|
|
|
storage_volume = max(storage_statistics, key=lambda x: storage_statistics[x]['free_percent'])
|
|
|
|
|
|
|
|
|
|
if storage_balancing_method == 'disk_io':
|
|
|
|
|
logging.info(f'{info_prefix} Getting most free storage volume by disk IO.')
|
|
|
|
|
logging.error(f'{error_prefix} Getting most free storage volume by disk IO is not yet supported.')
|
|
|
|
|
sys.exit(2)
|
|
|
|
|
|
|
|
|
|
return storage_volume
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __update_resource_statistics(resource_highest_used_resources_vm, resource_highest_free_resources_node, vm_statistics, node_statistics, balancing_method, balancing_mode):
|
|
|
|
|
""" Update VM and node resource statistics. """
|
|
|
|
|
info_prefix = 'Info: [rebalancing-resource-statistics-update]:'
|
|
|
|
|
@@ -607,7 +940,6 @@ def __update_resource_statistics(resource_highest_used_resources_vm, resource_hi
|
|
|
|
|
# Update dictionaries for new values
|
|
|
|
|
# Assign new rebalance node to vm
|
|
|
|
|
vm_statistics[vm_name]['node_rebalance'] = vm_node_rebalance
|
|
|
|
|
|
|
|
|
|
logging.info(f'Moving {vm_name} from {vm_node_parent} to {vm_node_rebalance}')
|
|
|
|
|
|
|
|
|
|
# Recalculate values for nodes
|
|
|
|
|
@@ -629,6 +961,39 @@ def __update_resource_statistics(resource_highest_used_resources_vm, resource_hi
|
|
|
|
|
return node_statistics, vm_statistics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __update_resource_storage_statistics(storage_statistics, resources_storage_most_free, vm_statistics, vm_name, vm_disk_device):
|
|
|
|
|
""" Update VM and storage resource statistics. """
|
|
|
|
|
info_prefix = 'Info: [rebalancing-storage-resource-statistics-update]:'
|
|
|
|
|
current_storage = vm_statistics[vm_name]['storages'][vm_disk_device]['storage_parent']
|
|
|
|
|
rebalance_storage = resources_storage_most_free
|
|
|
|
|
vm_storage_size = vm_statistics[vm_name]['storages'][vm_disk_device]['size'][:-1]
|
|
|
|
|
vm_storage_size_bytes = int(vm_storage_size) * 1024**3
|
|
|
|
|
|
|
|
|
|
# Assign new storage device to vm
|
|
|
|
|
logging.info(f'{info_prefix} Validating VM {vm_name} for potential storage rebalancing.')
|
|
|
|
|
if vm_statistics[vm_name]['storages'][vm_disk_device]['storage_rebalance'] is None and vm_statistics[vm_name]['storages'][vm_disk_device]['storage_rebalance'] != vm_statistics[vm_name]['storages'][vm_disk_device]['storage_parent']:
|
|
|
|
|
logging.info(f'{info_prefix} Setting VM {vm_name} from {current_storage} to {rebalance_storage} storage.')
|
|
|
|
|
vm_statistics[vm_name]['storages'][vm_disk_device]['storage_rebalance'] = resources_storage_most_free
|
|
|
|
|
else:
|
|
|
|
|
logging.info(f'{info_prefix} Setting VM {vm_name} from {current_storage} to {rebalance_storage} storage.')
|
|
|
|
|
|
|
|
|
|
# Recalculate values for storage
|
|
|
|
|
## Add freed resources to old parent storage device
|
|
|
|
|
storage_statistics[current_storage]['used'] = storage_statistics[current_storage]['used'] - vm_storage_size_bytes
|
|
|
|
|
storage_statistics[current_storage]['free'] = storage_statistics[current_storage]['free'] + vm_storage_size_bytes
|
|
|
|
|
storage_statistics[current_storage]['free_percent'] = (storage_statistics[current_storage]['free'] / storage_statistics[current_storage]['total']) * 100
|
|
|
|
|
storage_statistics[current_storage]['used_percent'] = (storage_statistics[current_storage]['used'] / storage_statistics[current_storage]['total']) * 100
|
|
|
|
|
|
|
|
|
|
## Removed newly allocated resources to new rebalanced storage device
|
|
|
|
|
storage_statistics[rebalance_storage]['used'] = storage_statistics[rebalance_storage]['used'] + vm_storage_size_bytes
|
|
|
|
|
storage_statistics[rebalance_storage]['free'] = storage_statistics[rebalance_storage]['free'] - vm_storage_size_bytes
|
|
|
|
|
storage_statistics[rebalance_storage]['free_percent'] = (storage_statistics[rebalance_storage]['free'] / storage_statistics[rebalance_storage]['total']) * 100
|
|
|
|
|
storage_statistics[rebalance_storage]['used_percent'] = (storage_statistics[rebalance_storage]['used'] / storage_statistics[rebalance_storage]['total']) * 100
|
|
|
|
|
|
|
|
|
|
logging.info(f'{info_prefix} Updated VM and storage statistics.')
|
|
|
|
|
return storage_statistics, vm_statistics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __get_vm_tags_include_groups(vm_statistics, node_statistics, balancing_method, balancing_mode):
|
|
|
|
|
""" Get VMs tags for include groups. """
|
|
|
|
|
info_prefix = 'Info: [rebalancing-tags-group-include]:'
|
|
|
|
|
@@ -703,27 +1068,88 @@ def __get_vm_tags_exclude_groups(vm_statistics, node_statistics, balancing_metho
|
|
|
|
|
return node_statistics, vm_statistics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __run_vm_rebalancing(api_object, vm_statistics_rebalanced, app_args):
|
|
|
|
|
def __wait_job_finalized(api_object, node_name, job_id, counter):
|
|
|
|
|
""" Wait for a job to be finalized. """
|
|
|
|
|
error_prefix = 'Error: [job-status-getter]:'
|
|
|
|
|
info_prefix = 'Info: [job-status-getter]:'
|
|
|
|
|
|
|
|
|
|
logging.info(f'{info_prefix} Getting job status for job {job_id}.')
|
|
|
|
|
task = api_object.nodes(node_name).tasks(job_id).status().get()
|
|
|
|
|
logging.info(f'{info_prefix} {task}')
|
|
|
|
|
|
|
|
|
|
if task['status'] == 'running':
|
|
|
|
|
logging.info(f'{info_prefix} Validating job {job_id} for the {counter} run.')
|
|
|
|
|
|
|
|
|
|
# Do not run for infinity this recursion and fail when reaching the limit.
|
|
|
|
|
if counter == 300:
|
|
|
|
|
logging.critical(f'{error_prefix} The job {job_id} on node {node_name} did not finished in time for migration.')
|
|
|
|
|
|
|
|
|
|
time.sleep(5)
|
|
|
|
|
counter = counter + 1
|
|
|
|
|
logging.info(f'{info_prefix} Revalidating job {job_id} in a next run.')
|
|
|
|
|
__wait_job_finalized(api_object, node_name, job_id, counter)
|
|
|
|
|
|
|
|
|
|
logging.info(f'{info_prefix} Job {job_id} for migration from {node_name} terminiated succesfully.')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __run_vm_rebalancing(api_object, vm_statistics, app_args, parallel_migrations):
|
|
|
|
|
""" Run & execute the VM rebalancing via API. """
|
|
|
|
|
error_prefix = 'Error: [rebalancing-executor]:'
|
|
|
|
|
info_prefix = 'Info: [rebalancing-executor]:'
|
|
|
|
|
|
|
|
|
|
if len(vm_statistics_rebalanced) > 0 and not app_args.dry_run:
|
|
|
|
|
for vm, value in vm_statistics_rebalanced.items():
|
|
|
|
|
if len(vm_statistics) > 0 and not app_args.dry_run:
|
|
|
|
|
for vm, value in vm_statistics.items():
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
# Migrate type VM (live migration).
|
|
|
|
|
if value['type'] == 'vm':
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing VM {vm} from node {value["node_parent"]} to node {value["node_rebalance"]}.')
|
|
|
|
|
api_object.nodes(value['node_parent']).qemu(value['vmid']).migrate().post(target=value['node_rebalance'],online=1)
|
|
|
|
|
job_id = api_object.nodes(value['node_parent']).qemu(value['vmid']).migrate().post(target=value['node_rebalance'],online=1)
|
|
|
|
|
|
|
|
|
|
# Migrate type CT (requires restart of container).
|
|
|
|
|
if value['type'] == 'ct':
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing CT {vm} from node {value["node_parent"]} to node {value["node_rebalance"]}.')
|
|
|
|
|
api_object.nodes(value['node_parent']).lxc(value['vmid']).migrate().post(target=value['node_rebalance'],restart=1)
|
|
|
|
|
job_id = api_object.nodes(value['node_parent']).lxc(value['vmid']).migrate().post(target=value['node_rebalance'],restart=1)
|
|
|
|
|
|
|
|
|
|
except proxmoxer.core.ResourceException as error_resource:
|
|
|
|
|
logging.critical(f'{error_prefix} {error_resource}')
|
|
|
|
|
|
|
|
|
|
# Wait for migration to be finished unless running parallel migrations.
|
|
|
|
|
if not bool(int(parallel_migrations)):
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing will be performed sequentially.')
|
|
|
|
|
__wait_job_finalized(api_object, value['node_parent'], job_id, counter=1)
|
|
|
|
|
else:
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing will be performed parallely.')
|
|
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
logging.info(f'{info_prefix} No rebalancing needed.')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __run_storage_rebalancing(api_object, vm_statistics, app_args, parallel_migrations):
|
|
|
|
|
""" Run & execute the storage rebalancing via API. """
|
|
|
|
|
error_prefix = 'Error: [rebalancing-executor]:'
|
|
|
|
|
info_prefix = 'Info: [rebalancing-executor]:'
|
|
|
|
|
|
|
|
|
|
if len(vm_statistics) > 0 and not app_args.dry_run:
|
|
|
|
|
for vm, value in vm_statistics.items():
|
|
|
|
|
for disk, disk_info in value['storages'].items():
|
|
|
|
|
|
|
|
|
|
if disk_info.get('storage_rebalance', None) is not None:
|
|
|
|
|
try:
|
|
|
|
|
# Migrate type VM (live migration).
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing storage of VM {vm} from node.')
|
|
|
|
|
job_id = api_object.nodes(value['node_parent']).qemu(value['vmid']).move_disk().post(disk=disk,storage=disk_info.get('storage_rebalance', None), delete=1)
|
|
|
|
|
|
|
|
|
|
except proxmoxer.core.ResourceException as error_resource:
|
|
|
|
|
logging.critical(f'{error_prefix} {error_resource}')
|
|
|
|
|
|
|
|
|
|
# Wait for migration to be finished unless running parallel migrations.
|
|
|
|
|
if not bool(int(parallel_migrations)):
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing will be performed sequentially.')
|
|
|
|
|
__wait_job_finalized(api_object, value['node_parent'], job_id, counter=1)
|
|
|
|
|
else:
|
|
|
|
|
logging.info(f'{info_prefix} Rebalancing will be performed parallely.')
|
|
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
logging.info(f'{info_prefix} No rebalancing needed.')
|
|
|
|
|
|
|
|
|
|
@@ -750,9 +1176,10 @@ def __create_cli_output(vm_statistics_rebalanced, app_args):
|
|
|
|
|
info_prefix = info_prefix_run
|
|
|
|
|
logging.info(f'{info_prefix} Start rebalancing vms to their new nodes.')
|
|
|
|
|
|
|
|
|
|
vm_to_node_list.append(['VM', 'Current Node', 'Rebalanced Node', 'VM Type'])
|
|
|
|
|
vm_to_node_list.append(['VM', 'Current Node', 'Rebalanced Node', 'Current Storage', 'Rebalanced Storage', 'VM Type'])
|
|
|
|
|
for vm_name, vm_values in vm_statistics_rebalanced.items():
|
|
|
|
|
vm_to_node_list.append([vm_name, vm_values['node_parent'], vm_values['node_rebalance'], vm_values['type']])
|
|
|
|
|
for disk, disk_info in vm_values['storages'].items():
|
|
|
|
|
vm_to_node_list.append([vm_name, vm_values['node_parent'], vm_values['node_rebalance'], disk_info.get('storage_parent', 'NA'), disk_info.get('storage_rebalance', 'NA'), vm_values['type']])
|
|
|
|
|
|
|
|
|
|
if len(vm_statistics_rebalanced) > 0:
|
|
|
|
|
logging.info(f'{info_prefix} Printing cli output of VM rebalancing.')
|
|
|
|
|
@@ -784,11 +1211,21 @@ def __print_table_cli(table, dry_run=False):
|
|
|
|
|
logging.info(f'{info_prefix} {row_format.format(*row)}')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def run_vm_rebalancing(api_object, vm_statistics_rebalanced, app_args):
|
|
|
|
|
def run_rebalancing(api_object, vm_statistics, app_args, parallel_migrations, balance_type):
|
|
|
|
|
""" Run rebalancing of vms to new nodes in cluster. """
|
|
|
|
|
__run_vm_rebalancing(api_object, vm_statistics_rebalanced, app_args)
|
|
|
|
|
__create_json_output(vm_statistics_rebalanced, app_args)
|
|
|
|
|
__create_cli_output(vm_statistics_rebalanced, app_args)
|
|
|
|
|
error_prefix = 'Error: [rebalancing-executor]:'
|
|
|
|
|
info_prefix = 'Info: [rebalancing-executor]:'
|
|
|
|
|
|
|
|
|
|
if balance_type == 'vm':
|
|
|
|
|
logging.info(f'{info_prefix} Starting executor for type vm.')
|
|
|
|
|
__run_vm_rebalancing(api_object, vm_statistics, app_args, parallel_migrations)
|
|
|
|
|
|
|
|
|
|
if balance_type == 'storage':
|
|
|
|
|
logging.info(f'{info_prefix} Starting executor for type vm.')
|
|
|
|
|
__run_storage_rebalancing(api_object, vm_statistics, app_args, parallel_migrations)
|
|
|
|
|
|
|
|
|
|
__create_json_output(vm_statistics, app_args)
|
|
|
|
|
__create_cli_output(vm_statistics, app_args)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
|
@@ -800,8 +1237,7 @@ def main():
|
|
|
|
|
pre_validations(config_path)
|
|
|
|
|
|
|
|
|
|
# Parse global config.
|
|
|
|
|
proxmox_api_host, proxmox_api_user, proxmox_api_pass, proxmox_api_ssl_v, balancing_method, balancing_mode, balancing_mode_option, balancing_type, \
|
|
|
|
|
balanciness, ignore_nodes, ignore_vms, daemon, schedule, log_verbosity = initialize_config_options(config_path)
|
|
|
|
|
proxmox_api_host, proxmox_api_user, proxmox_api_pass, proxmox_api_ssl_v, vm_balancing_enable, vm_balancing_method, vm_balancing_mode, vm_balancing_mode_option, vm_balancing_type, vm_balanciness, vm_parallel_migrations, vm_ignore_nodes, vm_ignore_vms, storage_balancing_enable, storage_balancing_method, storage_balancing_balanciness, storage_parallel_migrations, master_only, daemon, schedule, log_verbosity = initialize_config_options(config_path)
|
|
|
|
|
|
|
|
|
|
# Overwrite logging handler with user defined log verbosity.
|
|
|
|
|
initialize_logger(log_verbosity, update_log_verbosity=True)
|
|
|
|
|
@@ -810,17 +1246,33 @@ def main():
|
|
|
|
|
# API Authentication.
|
|
|
|
|
api_object = api_connect(proxmox_api_host, proxmox_api_user, proxmox_api_pass, proxmox_api_ssl_v)
|
|
|
|
|
|
|
|
|
|
# Get master node of cluster and ensure that ProxLB is only performed on the
|
|
|
|
|
# cluster master node to avoid ongoing rebalancing.
|
|
|
|
|
cluster_master, master_only = execute_rebalancing_only_by_master(api_object, master_only)
|
|
|
|
|
|
|
|
|
|
# Validate daemon service and skip following tasks when not being the cluster master.
|
|
|
|
|
if not cluster_master and master_only:
|
|
|
|
|
validate_daemon(daemon, schedule)
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
# Get metric & statistics for vms and nodes.
|
|
|
|
|
node_statistics = get_node_statistics(api_object, ignore_nodes)
|
|
|
|
|
vm_statistics = get_vm_statistics(api_object, ignore_vms, balancing_type)
|
|
|
|
|
node_statistics = update_node_statistics(node_statistics, vm_statistics)
|
|
|
|
|
node_statistics = get_node_statistics(api_object, vm_ignore_nodes)
|
|
|
|
|
vm_statistics = get_vm_statistics(api_object, vm_ignore_vms, vm_balancing_type)
|
|
|
|
|
storage_statistics = get_storage_statistics(api_object)
|
|
|
|
|
node_statistics = update_node_statistics(node_statistics, vm_statistics)
|
|
|
|
|
|
|
|
|
|
# Calculate rebalancing of vms.
|
|
|
|
|
node_statistics_rebalanced, vm_statistics_rebalanced = balancing_calculations(balancing_method, balancing_mode, balancing_mode_option,
|
|
|
|
|
node_statistics, vm_statistics, balanciness, rebalance=False, processed_vms=[])
|
|
|
|
|
vm_balancing_enable = False
|
|
|
|
|
if vm_balancing_enable:
|
|
|
|
|
node_statistics, vm_statistics = balancing_vm_calculations(vm_balancing_method, vm_balancing_mode, vm_balancing_mode_option, node_statistics, vm_statistics, vm_balanciness, rebalance=False, processed_vms=[])
|
|
|
|
|
run_rebalancing(api_object, vm_statistics, app_args, vm_parallel_migrations, balance_type='vm')
|
|
|
|
|
|
|
|
|
|
# Rebalance vms to new nodes within the cluster.
|
|
|
|
|
run_vm_rebalancing(api_object, vm_statistics_rebalanced, app_args)
|
|
|
|
|
# Calculate rebalancing of storage.
|
|
|
|
|
storage_balancing_enable = True
|
|
|
|
|
#storage_balancing_method = 'disk_space'
|
|
|
|
|
if storage_balancing_enable:
|
|
|
|
|
storage_statistics, vm_statistics = balancing_storage_calculations(storage_balancing_method, storage_statistics, vm_statistics, storage_balancing_balanciness, rebalance=False, processed_vms=[])
|
|
|
|
|
run_rebalancing(api_object, vm_statistics, app_args, storage_parallel_migrations, balance_type='storage')
|
|
|
|
|
|
|
|
|
|
# Validate for any errors.
|
|
|
|
|
post_validations()
|
|
|
|
|
|
