mirror of
https://github.com/gyptazy/ProxLB.git
synced 2026-04-06 04:41:58 +02:00
Compare commits
121 Commits
feature/28
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c39301ca96 | ||
|
|
b7a6fcec0c | ||
|
|
9966fbb13f | ||
|
|
e6ae357838 | ||
|
|
65b1bd5fee | ||
|
|
72283d8c19 | ||
|
|
4f85feacde | ||
|
|
34e340c25c | ||
|
|
da193f9d27 | ||
|
|
f11ca263b8 | ||
|
|
0af770c9df | ||
|
|
a9d11daf40 | ||
|
|
ab7ee0d687 | ||
|
|
e841481fdd | ||
|
|
5b7cc6727f | ||
|
|
15a05d320e | ||
|
|
e0331e83e1 | ||
|
|
ffd74d47e9 | ||
|
|
89ad425243 | ||
|
|
2ce3d73262 | ||
|
|
b8093454d7 | ||
|
|
d7631ef8f5 | ||
|
|
d546036a9a | ||
|
|
09b5b83c24 | ||
|
|
8d61ccfbb1 | ||
|
|
b39c13e2a5 | ||
|
|
8e759b778c | ||
|
|
22406e3628 | ||
|
|
e7f5d5142e | ||
|
|
48d621a06d | ||
|
|
c133ef1aee | ||
|
|
9ea04f904d | ||
|
|
5101202f72 | ||
|
|
929390b288 | ||
|
|
d4560c3af4 | ||
|
|
55c885194e | ||
|
|
3d9f0eb85e | ||
|
|
490fb55ee1 | ||
|
|
a70330d4c3 | ||
|
|
71d373eedb | ||
|
|
040eeb9f13 | ||
|
|
4ef1e92aad | ||
|
|
7e5fe13dfe | ||
|
|
66c2ab6570 | ||
|
|
ba63514896 | ||
|
|
571025a8a6 | ||
|
|
dd13181cf9 | ||
|
|
37d19a6a2d | ||
|
|
fe333749ce | ||
|
|
8f9bcfcdcf | ||
|
|
ff5fd2f7f1 | ||
|
|
1f6576ecd6 | ||
|
|
46bbe01141 | ||
|
|
07ed12fcb7 | ||
|
|
546fbc7d73 | ||
|
|
15436c431f | ||
|
|
33f6ff8db0 | ||
|
|
84628f232e | ||
|
|
6a91afd405 | ||
|
|
909643a09f | ||
|
|
7de1ba366b | ||
|
|
0cb19fab34 | ||
|
|
972b10b7e5 | ||
|
|
7fa110e465 | ||
|
|
948df0316b | ||
|
|
016378e37c | ||
|
|
8a193b9891 | ||
|
|
30e3b66be9 | ||
|
|
b9be405194 | ||
|
|
ac108f2abe | ||
|
|
02b43d3ef7 | ||
|
|
581d6d480b | ||
|
|
5b395b7f15 | ||
|
|
7d94c52883 | ||
|
|
7d19788be1 | ||
|
|
0bbc5992ca | ||
|
|
a4a5d9e68a | ||
|
|
af98ee8d5b | ||
|
|
afc93f7b21 | ||
|
|
bc6d8c8509 | ||
|
|
6d50f32486 | ||
|
|
5fe49a9dc1 | ||
|
|
fca1d1211c | ||
|
|
36388d9429 | ||
|
|
3f424e9e6d | ||
|
|
44a733aed3 | ||
|
|
2f44ff48a0 | ||
|
|
7b6db9cfdd | ||
|
|
8c473b416c | ||
|
|
51c8afe5c5 | ||
|
|
a8a154abde | ||
|
|
554a3eaf72 | ||
|
|
0b35987403 | ||
|
|
d93048db69 | ||
|
|
2aba7dbe23 | ||
|
|
ba388dfd7c | ||
|
|
5aa8257d40 | ||
|
|
99fefe20bf | ||
|
|
b9fb3a60e1 | ||
|
|
88b3288eb7 | ||
|
|
fa0113f112 | ||
|
|
0039ae9093 | ||
|
|
e3bbf31fdd | ||
|
|
bf393c6bbf | ||
|
|
7e5b72cfc7 | ||
|
|
0ba76f80f3 | ||
|
|
b48ff9d677 | ||
|
|
b5c11af474 | ||
|
|
af2992747d | ||
|
|
fb8dc40c16 | ||
|
|
34f1de8367 | ||
|
|
0e992e99de | ||
|
|
f5d073dc02 | ||
|
|
70ba1f2dfc | ||
|
|
c9855f1991 | ||
|
|
9bd29158b9 | ||
|
|
1ff0c5d96e | ||
|
|
3eb4038723 | ||
|
|
47e7dd3c56 | ||
|
|
bb8cf9033d | ||
|
|
756b4efcbd |
2
.changelogs/1.1.10/335-prevalidate-affinity-matrix.yml
Normal file
2
.changelogs/1.1.10/335-prevalidate-affinity-matrix.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
added:
|
||||
- Prevent redundant rebalancing by validating existing affinity enforcement before taking actions (@gyptazy). [#335]
|
||||
@@ -0,0 +1,2 @@
|
||||
added:
|
||||
- Add safety-guard for PVE 8 users when activating conntrack-aware migrations mistakenly (@gyptazy). [#359]
|
||||
@@ -0,0 +1,3 @@
|
||||
fixed:
|
||||
- Fixed the Proxmox API connection validation which returned a false-positive logging message of timeouts (@gyptazy). [#361]
|
||||
- Refactored Proxmox API connection functions
|
||||
@@ -0,0 +1,2 @@
|
||||
fixed:
|
||||
- Fixed a crash during PVE resource pool enumeration by skipping members not having a 'name' property (@stefanoettl). [#368]
|
||||
1
.changelogs/1.1.10/release_meta.yml
Normal file
1
.changelogs/1.1.10/release_meta.yml
Normal file
@@ -0,0 +1 @@
|
||||
date: 2025-11-25
|
||||
@@ -0,0 +1,2 @@
|
||||
fixed:
|
||||
- Fixed missing overprovisioning safety guard to avoid node overprovisioning (@gyptazy). [#275]
|
||||
@@ -0,0 +1,2 @@
|
||||
fixed:
|
||||
- Fixed affinity matrix pre-validation by inverting validations (@Thalagyrt). [#335]
|
||||
@@ -0,0 +1,2 @@
|
||||
added:
|
||||
- Add resource reservation support for PVE nodes (@Chipmonk2). [#373]
|
||||
@@ -0,0 +1,3 @@
|
||||
changed:
|
||||
- Changed balancing and sorting behaviour (@gyptazy). [#378]
|
||||
- Balancing objects will be ordered by: count of objects in affinity-rules, followed by memory size
|
||||
@@ -0,0 +1,2 @@
|
||||
added:
|
||||
- Add possibility to sort and select balancing workloads by smaller/larger guest objects (@gyptazy). [#387]
|
||||
@@ -0,0 +1,3 @@
|
||||
added:
|
||||
- Add support for Proxmox's native HA (affinity/anti-affinity) rules (@gyptazy). [#391]
|
||||
- Add support for Proxmox's native HA (node-affinity) rules for pinning guests to nodes (@gyptazy). [#391]
|
||||
2
.changelogs/1.1.11/395_fix_pool_based_node_pinning.yml
Normal file
2
.changelogs/1.1.11/395_fix_pool_based_node_pinning.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
fixed:
|
||||
- Fixed pool based node pinning (@gyptazy). [#395]
|
||||
2
.changelogs/1.1.11/402_add_ha_job_validation.yml
Normal file
2
.changelogs/1.1.11/402_add_ha_job_validation.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
added:
|
||||
- Add HA job validation for migration jobs (@gytazy). [#402]
|
||||
@@ -0,0 +1,2 @@
|
||||
added:
|
||||
- Add support for configuring node-pinning strictness (default: true) within pools (@gyptazy). [#406]
|
||||
2
.changelogs/1.1.11/408_fix_moving_ignored_guests.yml
Normal file
2
.changelogs/1.1.11/408_fix_moving_ignored_guests.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
fixed:
|
||||
- Fixed that ignored VMs/CTs got moved to another node when being ignored (@gyptazy). [#408]
|
||||
2
.changelogs/1.1.11/414_add_pinning_enforcement.yml
Normal file
2
.changelogs/1.1.11/414_add_pinning_enforcement.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
added:
|
||||
- Add new option to enforce node/guest pinning even when cluster is balanced from a resource perspective (@gyptazy). [#414]
|
||||
1
.changelogs/1.1.11/release_meta.yml
Normal file
1
.changelogs/1.1.11/release_meta.yml
Normal file
@@ -0,0 +1 @@
|
||||
date: 2026-01-12
|
||||
@@ -0,0 +1,2 @@
|
||||
fixed:
|
||||
- Fix PSI based balancing which resulted in a Python KeyError (@gyptazy). [#420]
|
||||
1
.changelogs/1.1.12/release_meta.yml
Normal file
1
.changelogs/1.1.12/release_meta.yml
Normal file
@@ -0,0 +1 @@
|
||||
date: TBD
|
||||
2
.changelogs/1.1.6/290_validate_user_token_syntax.yml
Normal file
2
.changelogs/1.1.6/290_validate_user_token_syntax.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
added:
|
||||
- Add validation for provided API user token id to avoid confusions (@gyptazy). [#291]
|
||||
@@ -0,0 +1,2 @@
|
||||
fixed:
|
||||
- Fix stacktrace output when validating permissions on non existing users in Proxmox (@gyptazy). [#291]
|
||||
@@ -0,0 +1,3 @@
|
||||
fixed:
|
||||
- Fix Overprovisioning first node if anti_affinity_group has only one member (@MiBUl-eu). [#295]
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
fixed:
|
||||
- Validate for node presence when pinning guests to avoid crashing (@gyptazy). [#296]
|
||||
|
||||
@@ -1 +1 @@
|
||||
date: TBD
|
||||
date: 2025-09-04
|
||||
|
||||
2
.changelogs/1.1.7/304_add_graceful_shutdown_sigint.yml
Normal file
2
.changelogs/1.1.7/304_add_graceful_shutdown_sigint.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
added:
|
||||
- Add graceful shutdown for SIGINT (e.g., CTRL + C abort). (@gyptazy). [#304]
|
||||
@@ -0,0 +1,2 @@
|
||||
added:
|
||||
- Add conntrack state aware migrations of VMs (@gyptazy). [#305]
|
||||
2
.changelogs/1.1.7/308_fix_only_validate_valid_jobids.yml
Normal file
2
.changelogs/1.1.7/308_fix_only_validate_valid_jobids.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
fixed:
|
||||
- Fix crash when validating absent migration job ids. (@gyptazy). [#308]
|
||||
@@ -0,0 +1,2 @@
|
||||
fixed:
|
||||
- Fix guest object names are not being evaluated in debug log. (@gyptazy). [#310]
|
||||
1
.changelogs/1.1.7/release_meta.yml
Normal file
1
.changelogs/1.1.7/release_meta.yml
Normal file
@@ -0,0 +1 @@
|
||||
date: 2025-09-19
|
||||
3
.changelogs/1.1.8/317_container_image_non_root.yml
Normal file
3
.changelogs/1.1.8/317_container_image_non_root.yml
Normal file
@@ -0,0 +1,3 @@
|
||||
changed:
|
||||
- Container image does not run as root anymore (@mikaelkrantz945). [#317]
|
||||
- Container image uses venv for running ProxLB (@mikaelkrantz945). [#317]
|
||||
@@ -0,0 +1,2 @@
|
||||
fixed:
|
||||
- Fix API errors when using conntrack aware migration with older PVE versions (@gyptazy). [#318]
|
||||
2
.changelogs/1.1.8/329_add_log_prefix.yml
Normal file
2
.changelogs/1.1.8/329_add_log_prefix.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
fixed:
|
||||
- Add a static ProxLB prefix to the log output when used by journal handler (@gyptazy). [#329]
|
||||
1
.changelogs/1.1.8/release_meta.yml
Normal file
1
.changelogs/1.1.8/release_meta.yml
Normal file
@@ -0,0 +1 @@
|
||||
date: 2025-10-09
|
||||
5
.changelogs/1.1.9/337_add_pressure_based_balancing.yml
Normal file
5
.changelogs/1.1.9/337_add_pressure_based_balancing.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
added:
|
||||
- Add pressure (PSI) based balancing for memory, cpu, disk (req. PVE9 or greater) (@gyptazy). [#337]
|
||||
- Pressure (PSI) based balancing for nodes
|
||||
- Pressure (PSI) based balancing for guests
|
||||
- Add PVE version evaluation
|
||||
2
.changelogs/1.1.9/342_add_memory_balancing_threshold.yml
Normal file
2
.changelogs/1.1.9/342_add_memory_balancing_threshold.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
added:
|
||||
- Add an optional memory balancing threshold (@gyptazy). [#342]
|
||||
@@ -0,0 +1,2 @@
|
||||
added:
|
||||
- Add affinity/anti-affinity support by pools (@gyptazy). [#343]
|
||||
1
.changelogs/1.1.9/release_meta.yml
Normal file
1
.changelogs/1.1.9/release_meta.yml
Normal file
@@ -0,0 +1 @@
|
||||
date: 2025-10-30
|
||||
@@ -40,6 +40,19 @@ jobs:
|
||||
# Install dependencies
|
||||
apt-get update && \
|
||||
apt-get install -y python3 python3-setuptools debhelper dh-python python3-pip python3-stdeb python3-proxmoxer python3-requests python3-urllib3 devscripts python3-all && \
|
||||
|
||||
# Get base version from source code
|
||||
BASE_VERSION=\$(grep __version__ proxlb/utils/version.py | awk '{print \$3}' | tr -d '\"')
|
||||
echo \"Base version: \$BASE_VERSION\"
|
||||
|
||||
# Build full version with timestamp
|
||||
FULL_VERSION=\"\${BASE_VERSION}+$(date +%Y%m%d%H%M)\"
|
||||
echo \"Full version: \$FULL_VERSION\"
|
||||
|
||||
# Update debian/changelog with new version
|
||||
dch --force-bad-version -v \"\$FULL_VERSION\" \
|
||||
\"Automated GitHub Actions build on $(date -u +'%Y-%m-%d %H:%M UTC').\" && \
|
||||
|
||||
# Build package using stdeb / setuptools
|
||||
# python3 setup.py --command-packages=stdeb.command bdist_deb && \
|
||||
# Build native package
|
||||
@@ -58,6 +71,10 @@ jobs:
|
||||
integration-test-debian:
|
||||
needs: build-package-debian
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
debian_version: [bookworm, trixie]
|
||||
name: Integration Test on Debian ${{ matrix.debian_version }}
|
||||
steps:
|
||||
- name: Download Debian package artifact
|
||||
uses: actions/download-artifact@v4
|
||||
@@ -66,13 +83,18 @@ jobs:
|
||||
path: package/
|
||||
|
||||
- name: Set up Docker with Debian image
|
||||
run: docker pull debian:latest
|
||||
run: docker pull debian:${{ matrix.debian_version }}
|
||||
|
||||
- name: Install and test Debian package in Docker container
|
||||
run: |
|
||||
docker run --rm -v $(pwd)/package:/package -w /package debian:latest bash -c "
|
||||
apt-get update && \
|
||||
apt-get install -y systemd && \
|
||||
apt-get install -y ./proxlb*.deb && \
|
||||
python3 -c 'import proxlb; print(\"OK: Debian package successfully installed.\")'
|
||||
"
|
||||
docker run --rm \
|
||||
-v "$(pwd)/package:/package" \
|
||||
-w /package \
|
||||
debian:${{ matrix.debian_version }} \
|
||||
bash -c "
|
||||
set -e
|
||||
apt-get update
|
||||
apt-get install -y python3 systemd
|
||||
apt-get install -y ./proxlb*.deb
|
||||
python3 -c 'import proxlb; print(\"OK: Debian package successfully installed on ${{ matrix.debian_version }}.\")'
|
||||
"
|
||||
|
||||
95
CHANGELOG.md
95
CHANGELOG.md
@@ -5,6 +5,101 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [1.1.11] - 2026-01-12
|
||||
|
||||
### Added
|
||||
|
||||
- Add support for Proxmoxs native HA (affinity/anti-affinity) rules [beta] (@gyptazy). [#391]
|
||||
- Add support for Proxmox native HA (node-affinity) rules for pinning guests to nodes [beta] (@gyptazy). [#391]
|
||||
- Add resource reservation support for PVE nodes (@Chipmonk2). [#373]
|
||||
- Add possibility to sort and select balancing workloads by smaller/larger guest objects (@gyptazy). [#387]
|
||||
- Add HA job validation for migration jobs to fetch child jobs (@gytazy). [#402]
|
||||
- Add support for configuring node-pinning strictness (default: true) within pools to allow strict/prefer modes (@gyptazy). [#406]
|
||||
- Add new option to enforce node/guest pinning even when cluster is balanced from a resource perspective (@gyptazy). [#414]
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix missing overprovisioning safety guard to avoid node overprovisioning (@gyptazy). [#275]
|
||||
- Fix affinity matrix pre-validation by inverting validations (@Thalagyrt). [#335]
|
||||
- Fix pool based node pinning which expects a list (@gyptazy). [#395]
|
||||
- Fix that ignored VMs/CTs got moved to another node when being ignored (@gyptazy). [#408]
|
||||
|
||||
### Changed
|
||||
|
||||
- Change balancing and sorting behaviour (@gyptazy). [#378]
|
||||
- Balancing objects will be ordered by count of objects in affinity-rules, followed by memory size (@gyptazy). [#378]
|
||||
|
||||
## [1.1.10] - 2025-11-25
|
||||
|
||||
### Added
|
||||
|
||||
- Prevent redundant rebalancing by validating existing affinity enforcement before taking actions (@gyptazy). [#335]
|
||||
- Add safety-guard for PVE 8 users when activating conntrack-aware migrations mistakenly (@gyptazy). [#359]
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix the Proxmox API connection validation which returned a false-positive logging message of timeouts (@gyptazy). [#361]
|
||||
- Refactored Proxmox API connection functions (@gyptazy). [#361]
|
||||
- Fix a crash during PVE resource pool enumeration by skipping members not having a 'name' property (@stefanoettl). [#368]
|
||||
|
||||
## [1.1.9.1] - 2025-10-30
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix quoting in f-strings which may cause issues on PVE 8 / Debian Bookworm systems (@gyptazy). [#352]
|
||||
|
||||
## [1.1.9] - 2025-10-30
|
||||
|
||||
### Added
|
||||
|
||||
- Add an optional memory balancing threshold (@gyptazy). [#342]
|
||||
- Add affinity/anti-affinity support by pools (@gyptazy). [#343]
|
||||
- Add pressure (PSI) based balancing for memory, cpu, disk (req. PVE9 or greater) (@gyptazy). [#337]
|
||||
- Pressure (PSI) based balancing for nodes
|
||||
- Pressure (PSI) based balancing for guests
|
||||
- Add PVE version evaluation
|
||||
|
||||
## [1.1.8] - 2025-10-09
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix API errors when using conntrack aware migration with older PVE versions (@gyptazy). [#318]
|
||||
- Add a static ProxLB prefix to the log output when used by journal handler (@gyptazy). [#329]
|
||||
|
||||
### Changed
|
||||
- Container image does not run as root anymore (@mikaelkrantz945). [#317]
|
||||
- Container image uses venv for running ProxLB (@mikaelkrantz945). [#317]
|
||||
|
||||
## [1.1.7] - 2025-09-19
|
||||
|
||||
### Added
|
||||
|
||||
- Add conntrack state aware migrations of VMs (@gyptazy). [#305]
|
||||
- Add graceful shutdown for SIGINT (e.g., CTRL + C abort). (@gyptazy). [#304]
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix crash when validating absent migration job ids. (@gyptazy). [#308]
|
||||
- Fix guest object names are not being evaluated in debug log. (@gyptazy). [#310]
|
||||
|
||||
## [1.1.6.1] - 2025-09-04
|
||||
|
||||
### Fixed
|
||||
|
||||
- Validate for node presence when pinning VMs to avoid crashing (@gyptazy). [#296]
|
||||
|
||||
## [1.1.6] - 2025-09-04
|
||||
|
||||
### Added
|
||||
|
||||
- Add validation for provided API user token id to avoid confusions (@gyptazy). [#291]
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix stacktrace output when validating permissions on non existing users in Proxmox (@gyptazy). [#291]
|
||||
- Fix Overprovisioning first node if anti_affinity_group has only one member (@MiBUl-eu). [#295]
|
||||
- Validate for node presence when pinning guests to avoid crashing (@gyptazy). [#296]
|
||||
- Fix balancing evaluation of guest types (e.g., VM or CT) (@gyptazy). [#268]
|
||||
|
||||
## [1.1.5] - 2025-07-14
|
||||
|
||||
|
||||
35
Dockerfile
35
Dockerfile
@@ -9,20 +9,33 @@ LABEL org.label-schema.vendor="gyptazy"
|
||||
LABEL org.label-schema.url="https://proxlb.de"
|
||||
LABEL org.label-schema.vcs-url="https://github.com/gyptazy/ProxLB"
|
||||
|
||||
# Install Python3
|
||||
RUN apk add --no-cache python3 py3-pip
|
||||
# --- Step 1 (root): system deps, user, dirs ---
|
||||
RUN apk add --no-cache python3 py3-pip \
|
||||
&& addgroup -S plb \
|
||||
&& adduser -S -G plb -h /home/plb plb \
|
||||
&& mkdir -p /app/conf /opt/venv \
|
||||
&& chown -R plb:plb /app /home/plb /opt/venv
|
||||
|
||||
# Create a directory for the app
|
||||
WORKDIR /app
|
||||
|
||||
# Copy the python program from the current directory to /app
|
||||
COPY proxlb /app/proxlb
|
||||
# Copy only requirements first for better layer caching
|
||||
COPY --chown=plb:plb requirements.txt /app/requirements.txt
|
||||
|
||||
# Copy requirements to the container
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
# --- Step 2 (appuser): venv + deps + code ---
|
||||
USER plb
|
||||
|
||||
# Install dependencies in the virtual environment
|
||||
RUN pip install --break-system-packages -r /app/requirements.txt
|
||||
# Create venv owned by appuser and put it on PATH
|
||||
RUN python3 -m venv /opt/venv
|
||||
ENV PATH="/opt/venv/bin:${PATH}"
|
||||
|
||||
# Set the entry point to use the virtual environment's python
|
||||
ENTRYPOINT ["/usr/bin/python3", "/app/proxlb/main.py"]
|
||||
# Install Python dependencies into the venv (no PEP 668 issues)
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
# Copy application code (owned by appuser)
|
||||
COPY --chown=plb:plb proxlb /app/proxlb
|
||||
|
||||
# Optional: placeholder config so a bind-mount can override cleanly
|
||||
RUN touch /app/conf/proxlb.yaml
|
||||
|
||||
# Run as non-root using venv Python
|
||||
ENTRYPOINT ["/opt/venv/bin/python", "/app/proxlb/main.py"]
|
||||
|
||||
442
README.md
442
README.md
@@ -1,439 +1,7 @@
|
||||
# ProxLB - (Re)Balance VM Workloads in Proxmox Clusters
|
||||
<img align="left" src="https://cdn.gyptazy.com/images/Prox-LB-logo.jpg"/>
|
||||
<br>
|
||||
# ProxLB Moved
|
||||
|
||||
<p float="center"><img src="https://img.shields.io/github/license/gyptazy/ProxLB"/><img src="https://img.shields.io/github/contributors/gyptazy/ProxLB"/><img src="https://img.shields.io/github/last-commit/gyptazy/ProxLB/main"/><img src="https://img.shields.io/github/issues-raw/gyptazy/ProxLB"/><img src="https://img.shields.io/github/issues-pr/gyptazy/ProxLB"/></p>
|
||||
You can find the new location of the `ProxLB` project at:
|
||||
[github.com/credativ/ProxLB](https://github.com/credativ/ProxLB)
|
||||
|
||||
## Table of Contents
|
||||
1. [Introduction](#introduction)
|
||||
2. [Features](#features)
|
||||
3. [How does it work?](#how-does-it-work)
|
||||
4. [Installation](#installation)
|
||||
1. [Requirements / Dependencies](#requirements--dependencies)
|
||||
2. [Debian Package](#debian-package)
|
||||
4. [Container / Docker](#container--docker)
|
||||
5. [Source](#source)
|
||||
5. [Usage / Configuration](#usage--configuration)
|
||||
1. [GUI Integration](#gui-integration)
|
||||
2. [Proxmox HA Integration](#proxmox-ha-integration)
|
||||
3. [Options](#options)
|
||||
6. [Affinity & Anti-Affinity Rules](#affinity--anti-affinity-rules)
|
||||
1. [Affinity Rules](#affinity-rules)
|
||||
2. [Anti-Affinity Rules](#anti-affinity-rules)
|
||||
3. [Ignore VMs](#ignore-vms)
|
||||
4. [Pin VMs to Hypervisor Nodes](#pin-vms-to-hypervisor-nodes)
|
||||
7. [Maintenance](#maintenance)
|
||||
8. [Misc](#misc)
|
||||
1. [Bugs](#bugs)
|
||||
2. [Contributing](#contributing)
|
||||
3. [Documentation](#documentation)
|
||||
4. [Support](#support)
|
||||
9. [Author(s)](#authors)
|
||||
|
||||
|
||||
## Introduction
|
||||
ProxLB is an advanced load balancing solution specifically designed for Proxmox clusters, addressing the absence of a Dynamic Resource Scheduler (DRS) that is familiar to VMware users. As a third-party solution, ProxLB enhances the management and efficiency of Proxmox clusters by intelligently distributing workloads across available nodes. Workloads can be balanced by different times like the guest's memory, CPU or disk usage or their assignment to avoid overprovisioning and ensuring resources.
|
||||
|
||||
One of the key advantages of ProxLB is that it is fully open-source and free, making it accessible for anyone to use, modify, and contribute to. This ensures transparency and fosters community-driven improvements. ProxLB supports filtering and ignoring specific nodes and guests through configuration files and API calls, providing administrators with the flexibility to tailor the load balancing behavior to their specific needs.
|
||||
|
||||
A standout feature of ProxLB is its maintenance mode. When enabled, all guest workloads are automatically moved to other nodes within the cluster, ensuring that a node can be safely updated, rebooted, or undergo hardware maintenance without disrupting the overall cluster operation. Additionally, ProxLB supports both affinity and anti-affinity rules, allowing operators to group multiple guests to run together on the same node or ensure that certain guests do not run on the same node, depending on the cluster's node count. This feature is crucial for optimizing performance and maintaining high availability.
|
||||
|
||||
ProxLB can also return the best next node for guest placement, which can be integrated into CI/CD pipelines using tools like Ansible or Terraform. This capability streamlines the deployment process and ensures efficient resource utilization. Furthermore, ProxLB leverages the Proxmox API, including the entire ACL (Access Control List) system, for secure and efficient operation. Unlike some solutions, it does not require SSH access, enhancing security and simplifying configuration.
|
||||
|
||||
Overall, ProxLB significantly enhances resource management by intelligently distributing workloads, reducing downtime through its maintenance mode, and providing improved flexibility with affinity and anti-affinity rules. Its seamless integration with CI/CD tools and reliance on the Proxmox API make it a robust and secure solution for optimizing Proxmox cluster performance.
|
||||
|
||||
### Video of Migration
|
||||
<img src="https://cdn.gyptazy.com/images/proxlb-rebalancing-demo.gif"/>
|
||||
|
||||
## Features
|
||||
ProxLB's key features are by enabling automatic rebalancing of VMs and CTs across a Proxmox cluster based on memory, CPU, and local disk usage while identifying optimal nodes for automation. It supports maintenance mode, affinity rules, and seamless Proxmox API integration with ACL support, offering flexible usage as a one-time operation, a daemon, or through the Proxmox Web GUI.
|
||||
|
||||
**Features**
|
||||
* Rebalance VMs/CTs in the cluster by:
|
||||
* Memory
|
||||
* Disk (only local storage)
|
||||
* CPU
|
||||
* Get best nodes for further automation
|
||||
* Supported Guest Types
|
||||
* VMs
|
||||
* CTs
|
||||
* Maintenance Mode
|
||||
* Set node(s) into maintenance
|
||||
* Move all workloads to different nodes
|
||||
* Affinity / Anti-Affinity Rules
|
||||
* Fully based on Proxmox API
|
||||
* Fully integrated into the Proxmox ACL
|
||||
* No SSH required
|
||||
* Usage
|
||||
* One-Time
|
||||
* Daemon
|
||||
* Proxmox Web GUI Integration
|
||||
|
||||
## How does it work?
|
||||
ProxLB is a load-balancing system designed to optimize the distribution of virtual machines (VMs) and containers (CTs) across a cluster. It works by first gathering resource usage metrics from all nodes in the cluster through the Proxmox API. This includes detailed resource metrics for each VM and CT on every node. ProxLB then evaluates the difference between the maximum and minimum resource usage of the nodes, referred to as "Balanciness." If this difference exceeds a predefined threshold (which is configurable), the system initiates the rebalancing process.
|
||||
|
||||
Before starting any migrations, ProxLB validates that rebalancing actions are necessary and beneficial. Depending on the selected balancing mode — such as CPU, memory, or disk — it creates a balancing matrix. This matrix sorts the VMs by their maximum used or assigned resources, identifying the VM with the highest usage. ProxLB then places this VM on the node with the most free resources in the selected balancing type. This process runs recursively until the operator-defined Balanciness is achieved. Balancing can be defined for the used or max. assigned resources of VMs/CTs.
|
||||
|
||||
## Installation
|
||||
|
||||
### Requirements / Dependencies
|
||||
* Proxmox
|
||||
* Proxmox 7.x
|
||||
* Proxmox 8.x
|
||||
* Proxmox 9.x (Beta 1 tested)
|
||||
* Python3.x
|
||||
* proxmoxer
|
||||
* requests
|
||||
* urllib3
|
||||
* pyyaml
|
||||
|
||||
The dependencies can simply be installed with `pip` by running the following command:
|
||||
```
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
*Note: Distribution packages, such like the provided `.deb` package will automatically resolve and install all required dependencies by using already packaged version from the distribution's repository. By using the Docker (container) image or Debian packages, you do not need to take any care of the requirements listed here.*
|
||||
|
||||
### Debian Package
|
||||
ProxLB is a powerful and flexible load balancer designed to work across various architectures, including `amd64`, `arm64`, `rv64` and many other ones that support Python. It runs independently of the underlying hardware, making it a versatile choice for different environments. This chapter covers the step-by-step process to install ProxLB on Debian-based systems, including Debian clones like Ubuntu.
|
||||
|
||||
#### Quick-Start
|
||||
You can simply use this snippet to install the repository and to install ProxLB on your system.
|
||||
|
||||
```bash
|
||||
echo "deb https://repo.gyptazy.com/stable /" > /etc/apt/sources.list.d/proxlb.list
|
||||
wget -O /etc/apt/trusted.gpg.d/proxlb.asc https://repo.gyptazy.com/repository.gpg
|
||||
apt-get update && apt-get -y install proxlb
|
||||
cp /etc/proxlb/proxlb_example.yaml /etc/proxlb/proxlb.yaml
|
||||
# Adjust the config to your needs
|
||||
vi /etc/proxlb/proxlb.yaml
|
||||
systemctl start proxlb
|
||||
```
|
||||
|
||||
Afterwards, ProxLB is running in the background and balances your cluster by your defined balancing method (default: memory).
|
||||
|
||||
#### Details
|
||||
ProxLB provides two different repositories:
|
||||
* https://repo.gyptazy.com/stable (only stable release)
|
||||
* https://repo.gyptazy.com/testing (bleeding edge - not recommended)
|
||||
|
||||
The repository is signed and the GPG key can be found at:
|
||||
* https://repo.gyptazy.com/repository.gpg
|
||||
|
||||
You can also simply import it by running:
|
||||
|
||||
```
|
||||
# KeyID: 17169F23F9F71A14AD49EDADDB51D3EB01824F4C
|
||||
# UID: gyptazy Solutions Repository <contact@gyptazy.com>
|
||||
# SHA256: 52c267e6f4ec799d40cdbdb29fa518533ac7942dab557fa4c217a76f90d6b0f3 repository.gpg
|
||||
|
||||
wget -O /etc/apt/trusted.gpg.d/proxlb.asc https://repo.gyptazy.com/repository.gpg
|
||||
```
|
||||
|
||||
*Note: The defined repositories `repo.gyptazy.com` and `repo.proxlb.de` are the same!*
|
||||
|
||||
#### Debian Packages (.deb files)
|
||||
If you do not want to use the repository you can also find the debian packages as a .deb file on gyptazy's CDN at:
|
||||
* https://cdn.gyptazy.com/files/os/debian/proxlb/
|
||||
|
||||
Afterwards, you can simply install the package by running:
|
||||
```bash
|
||||
dpkg -i proxlb_*.deb
|
||||
cp /etc/proxlb/proxlb_example.yaml /etc/proxlb/proxlb.yaml
|
||||
# Adjust the config to your needs
|
||||
vi /etc/proxlb/proxlb.yaml
|
||||
systemctl start proxlb
|
||||
```
|
||||
|
||||
### Container Images / Docker
|
||||
Using the ProxLB container images is straight forward and only requires you to mount the config file.
|
||||
|
||||
```bash
|
||||
# Pull the image
|
||||
docker pull cr.gyptazy.com/proxlb/proxlb:latest
|
||||
# Download the config
|
||||
wget -O proxlb.yaml https://raw.githubusercontent.com/gyptazy/ProxLB/refs/heads/main/config/proxlb_example.yaml
|
||||
# Adjust the config to your needs
|
||||
vi proxlb.yaml
|
||||
# Start the ProxLB container image with the ProxLB config
|
||||
docker run -it --rm -v $(pwd)/proxlb.yaml:/etc/proxlb/proxlb.yaml proxlb
|
||||
```
|
||||
|
||||
*Note: ProxLB container images are officially only available at cr.proxlb.de and cr.gyptazy.com.*
|
||||
|
||||
#### Overview of Images
|
||||
| Version | Image |
|
||||
|------|:------:|
|
||||
| latest | cr.gyptazy.com/proxlb/proxlb:latest |
|
||||
| v1.1.5 | cr.gyptazy.com/proxlb/proxlb:v1.1.5 |
|
||||
| v1.1.4 | cr.gyptazy.com/proxlb/proxlb:v1.1.4 |
|
||||
| v1.1.3 | cr.gyptazy.com/proxlb/proxlb:v1.1.3 |
|
||||
| v1.1.2 | cr.gyptazy.com/proxlb/proxlb:v1.1.2 |
|
||||
| v1.1.1 | cr.gyptazy.com/proxlb/proxlb:v1.1.1 |
|
||||
| v1.1.0 | cr.gyptazy.com/proxlb/proxlb:v1.1.0 |
|
||||
| v1.0.6 | cr.gyptazy.com/proxlb/proxlb:v1.0.6 |
|
||||
| v1.0.5 | cr.gyptazy.com/proxlb/proxlb:v1.0.5 |
|
||||
| v1.0.4 | cr.gyptazy.com/proxlb/proxlb:v1.0.4 |
|
||||
| v1.0.3 | cr.gyptazy.com/proxlb/proxlb:v1.0.3 |
|
||||
| v1.0.2 | cr.gyptazy.com/proxlb/proxlb:v1.0.2 |
|
||||
| v1.0.0 | cr.gyptazy.com/proxlb/proxlb:v1.0.0 |
|
||||
| v0.9.9 | cr.gyptazy.com/proxlb/proxlb:v0.9.9 |
|
||||
|
||||
### Source
|
||||
ProxLB can also easily be used from the provided sources - for traditional systems but also as a Docker/Podman container image.
|
||||
|
||||
#### Traditional System
|
||||
Setting up and running ProxLB from the sources is simple and requires just a few commands. Ensure Python 3 and the Python dependencies are installed on your system, then run ProxLB using the following command:
|
||||
```bash
|
||||
git clone https://github.com/gyptazy/ProxLB.git
|
||||
cd ProxLB
|
||||
```
|
||||
|
||||
Afterwards simply adjust the config file to your needs:
|
||||
```bash
|
||||
vi config/proxlb.yaml
|
||||
```
|
||||
|
||||
Start ProxLB by Python3 on the system:
|
||||
```bash
|
||||
python3 proxlb/main.py -c config/proxlb.yaml
|
||||
```
|
||||
|
||||
#### Container Image
|
||||
Creating a container image of ProxLB is straightforward using the provided Dockerfile. The Dockerfile simplifies the process by automating the setup and configuration required to get ProxLB running in an Alpine container. Simply follow the steps in the Dockerfile to build the image, ensuring all dependencies and configurations are correctly applied. For those looking for an even quicker setup, a ready-to-use ProxLB container image is also available, eliminating the need for manual building and allowing for immediate deployment.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/gyptazy/ProxLB.git
|
||||
cd ProxLB
|
||||
docker build -t proxlb .
|
||||
```
|
||||
|
||||
Afterwards simply adjust the config file to your needs:
|
||||
```bash
|
||||
vi config/proxlb.yaml
|
||||
```
|
||||
|
||||
Finally, start the created container.
|
||||
```bash
|
||||
docker run -it --rm -v $(pwd)/proxlb.yaml:/etc/proxlb/proxlb.yaml proxlb
|
||||
```
|
||||
|
||||
## Usage / Configuration
|
||||
Running ProxLB is straightforward and versatile, as it only requires `Python3` and the `proxmoxer` library. This means ProxLB can be executed directly on a Proxmox node or on dedicated systems such as Debian, RedHat, or even FreeBSD, provided that the Proxmox API is accessible from the client running ProxLB. ProxLB can also run inside a Container - Docker or LXC - and is simply up to you.
|
||||
|
||||
### GUI Integration
|
||||
<img align="left" src="https://cdn.gyptazy.com/images/proxlb-GUI-integration.jpg"/> ProxLB can also be accessed through the Proxmox Web UI by installing the optional `pve-proxmoxlb-service-ui` package, which depends on the proxlb package. For full Web UI integration, this package must be installed on all nodes within the cluster. Once installed, a new menu item - `Rebalancing`, appears in the cluster level under the HA section. Once installed, it offers two key functionalities:
|
||||
* Rebalancing VM workloads
|
||||
* Migrate VM workloads away from a defined node (e.g. maintenance preparation)
|
||||
|
||||
**Note:** This package is currently discontinued and will be readded at a later time. See also: [#44: How to install pve-proxmoxlb-service-ui package](https://github.com/gyptazy/ProxLB/issues/44).
|
||||
|
||||
### Proxmox HA Integration
|
||||
Proxmox HA (High Availability) groups are designed to ensure that virtual machines (VMs) remain running within a Proxmox cluster. HA groups define specific rules for where VMs should be started or migrated in case of node failures, ensuring minimal downtime and automatic recovery.
|
||||
|
||||
However, when used in conjunction with ProxLB, the built-in load balancer for Proxmox, conflicts can arise. ProxLB operates with its own logic for workload distribution, taking into account affinity and anti-affinity rules. While it effectively balances guest workloads, it may re-shift and redistribute VMs in a way that does not align with HA group constraints, potentially leading to unsuitable placements.
|
||||
|
||||
Due to these conflicts, it is currently not recommended to use both HA groups and ProxLB simultaneously. The interaction between the two mechanisms can lead to unexpected behavior, where VMs might not adhere to HA group rules after being moved by ProxLB.
|
||||
|
||||
A solution to improve compatibility between HA groups and ProxLB is under evaluation, aiming to ensure that both features can work together without disrupting VM placement strategies.
|
||||
|
||||
See also: [#65: Host groups: Honour HA groups](https://github.com/gyptazy/ProxLB/issues/65).
|
||||
|
||||
### Options
|
||||
The following options can be set in the configuration file `proxlb.yaml`:
|
||||
|
||||
| Section | Option | Sub Option | Example | Type | Description |
|
||||
|---------|:------:|:----------:|:-------:|:----:|:-----------:|
|
||||
| `proxmox_api` | | | | | |
|
||||
| | hosts | | ['virt01.example.com', '10.10.10.10', 'fe01:bad:code::cafe', 'virt01.example.com:443', '[fc00::1]', '[fc00::1]:443', 'fc00::1:8006'] | `List` | List of Proxmox nodes. Can be IPv4, IPv6 or mixed. You can specify custom ports. In case of IPv6 without brackets the port is considered after the last colon |
|
||||
| | user | | root@pam | `Str` | Username for the API. |
|
||||
| | pass | | FooBar | `Str` | Password for the API. (Recommended: Use API token authorization!) |
|
||||
| | token_id | | proxlb | `Str` | Token ID of the user for the API. |
|
||||
| | token_secret | | 430e308f-1337-1337-beef-1337beefcafe | `Str` | Secret of the token ID for the API. |
|
||||
| | ssl_verification | | True | `Bool` | Validate SSL certificates (1) or ignore (0). [values: `1` (default), `0`] |
|
||||
| | timeout | | 10 | `Int` | Timeout for the Proxmox API in sec. |
|
||||
| | retries | | 1 | `Int` | How often a connection attempt to the defined API host should be performed. |
|
||||
| | wait_time | | 1 | `Int` | How many seconds should be waited before performing another connection attempt to the API host. |
|
||||
| `proxmox_cluster` | | | | | |
|
||||
| | maintenance_nodes | | ['virt66.example.com'] | `List` | A list of Proxmox nodes that are defined to be in a maintenance. (must be the same node names as used within the cluster) |
|
||||
| | ignore_nodes | | [] | `List` | A list of Proxmox nodes that are defined to be ignored. |
|
||||
| | overprovisioning | | False | `Bool` | Avoids balancing when nodes would become overprovisioned. |
|
||||
| `balancing` | | | | | |
|
||||
| | enable | | True | `Bool` | Enables the guest balancing.|
|
||||
| | enforce_affinity | | True | `Bool` | Enforcing affinity/anti-affinity rules but balancing might become worse. |
|
||||
| | parallel | | False | `Bool` | If guests should be moved in parallel or sequentially.|
|
||||
| | parallel_jobs | | 5 | `Int` | The amount if parallel jobs when migrating guests. (default: `5`)|
|
||||
| | live | | True | `Bool` | If guests should be moved live or shutdown.|
|
||||
| | with_local_disks | | True | `Bool` | If balancing of guests should include local disks.|
|
||||
| | balance_types | | ['vm', 'ct'] | `List` | Defined the types of guests that should be honored. [values: `vm`, `ct`]|
|
||||
| | max_job_validation | | 1800 | `Int` | How long a job validation may take in seconds. (default: 1800) |
|
||||
| | balanciness | | 10 | `Int` | The maximum delta of resource usage between node with highest and lowest usage. |
|
||||
| | method | | memory | `Str` | The balancing method that should be used. [values: `memory` (default), `cpu`, `disk`]|
|
||||
| | mode | | used | `Str` | The balancing mode that should be used. [values: `used` (default), `assigned`] |
|
||||
| `service` | | | | | |
|
||||
| | daemon | | True | `Bool` | If daemon mode should be activated. |
|
||||
| | `schedule` | | | `Dict` | Schedule config block for rebalancing. |
|
||||
| | | interval | 12 | `Int` | How often rebalancing should occur in daemon mode.|
|
||||
| | | format | hours | `Str` | Sets the time format. [values: `hours` (default), `minutes`]|
|
||||
| | `delay` | | | `Dict` | Schedule config block for an optional delay until the service starts. |
|
||||
| | | enable | False | `Bool` | If a delay time should be validated.|
|
||||
| | | time | 1 | `Int` | Delay time until the service starts after the initial execution.|
|
||||
| | | format | hours | `Str` | Sets the time format. [values: `hours` (default), `minutes`]|
|
||||
| | log_level | | INFO | `Str` | Defines the default log level that should be logged. [values: `INFO` (default), `WARNING`, `CRITICAL`, `DEBUG`] |
|
||||
|
||||
|
||||
An example of the configuration file looks like:
|
||||
```
|
||||
proxmox_api:
|
||||
hosts: ['virt01.example.com', '10.10.10.10', 'fe01:bad:code::cafe']
|
||||
user: root@pam
|
||||
pass: crazyPassw0rd!
|
||||
# API Token method
|
||||
# token_id: proxlb
|
||||
# token_secret: 430e308f-1337-1337-beef-1337beefcafe
|
||||
ssl_verification: True
|
||||
timeout: 10
|
||||
# API Connection retries
|
||||
# retries: 1
|
||||
# wait_time: 1
|
||||
|
||||
proxmox_cluster:
|
||||
maintenance_nodes: ['virt66.example.com']
|
||||
ignore_nodes: []
|
||||
overprovisioning: True
|
||||
|
||||
balancing:
|
||||
enable: True
|
||||
enforce_affinity: False
|
||||
parallel: False
|
||||
live: True
|
||||
with_local_disks: True
|
||||
balance_types: ['vm', 'ct']
|
||||
max_job_validation: 1800
|
||||
balanciness: 5
|
||||
method: memory
|
||||
mode: used
|
||||
|
||||
service:
|
||||
daemon: True
|
||||
schedule:
|
||||
interval: 12
|
||||
format: hours
|
||||
delay:
|
||||
enable: False
|
||||
time: 1
|
||||
format: hours
|
||||
log_level: INFO
|
||||
```
|
||||
|
||||
### Parameters
|
||||
The following options and parameters are currently supported:
|
||||
|
||||
| Option | Long Option | Description | Default |
|
||||
|------|:------:|------:|------:|
|
||||
| -c | --config | Path to a config file. | /etc/proxlb/proxlb.yaml (default) |
|
||||
| -d | --dry-run | Performs a dry-run without doing any actions. | False |
|
||||
| -j | --json | Returns a JSON of the VM movement. | False |
|
||||
| -b | --best-node | Returns the best next node for a VM/CT placement (useful for further usage with Terraform/Ansible). | False |
|
||||
| -v | --version | Returns the ProxLB version on stdout. | False |
|
||||
|
||||
## Affinity & Anti-Affinity Rules
|
||||
ProxLB provides an advanced mechanism to define affinity and anti-affinity rules, enabling precise control over virtual machine (VM) placement. These rules help manage resource distribution, improve high availability configurations, and optimize performance within a Proxmox Virtual Environment (PVE) cluster. By leveraging Proxmox’s integrated access management, ProxLB ensures that users can only define and manage rules for guests they have permission to access.
|
||||
|
||||
ProxLB implements affinity and anti-affinity rules through a tag-based system within the Proxmox web interface. Each guest (virtual machine or container) can be assigned specific tags, which then dictate its placement behavior. This method maintains a streamlined and secure approach to managing VM relationships while preserving Proxmox’s inherent permission model.
|
||||
|
||||
### Affinity Rules
|
||||
<img align="left" src="https://cdn.gyptazy.com/images/proxlb-affinity-rules.jpg"/> Affinity rules are used to group certain VMs together, ensuring that they run on the same host whenever possible. This can be beneficial for workloads requiring low-latency communication, such as clustered databases or application servers that frequently exchange data.
|
||||
|
||||
To define an affinity rule which keeps all guests assigned to this tag together on a node, users assign a tag with the prefix `plb_affinity_$TAG`:
|
||||
|
||||
#### Example for Screenshot
|
||||
```
|
||||
plb_affinity_talos
|
||||
```
|
||||
|
||||
As a result, ProxLB will attempt to place all VMs with the `plb_affinity_web` tag on the same host (see also the attached screenshot with the same node).
|
||||
|
||||
### Anti-Affinity Rules
|
||||
<img align="left" src="https://cdn.gyptazy.com/images/proxlb-anti-affinity-rules.jpg"/> Conversely, anti-affinity rules ensure that designated VMs do not run on the same physical host. This is particularly useful for high-availability setups, where redundancy is crucial. Ensuring that critical services are distributed across multiple hosts reduces the risk of a single point of failure.
|
||||
|
||||
To define an anti-affinity rule that ensures to not move systems within this group to the same node, users assign a tag with the prefix:
|
||||
|
||||
#### Example for Screenshot
|
||||
```
|
||||
plb_anti_affinity_ntp
|
||||
```
|
||||
|
||||
As a result, ProxLB will try to place the VMs with the `plb_anti_affinity_ntp` tag on different hosts (see also the attached screenshot with the different nodes).
|
||||
|
||||
**Note:** While this ensures that ProxLB tries distribute these VMs across different physical hosts within the Proxmox cluster this may not always work. If you have more guests attached to the group than nodes in the cluster, we still need to run them anywhere. If this case occurs, the next one with the most free resources will be selected.
|
||||
|
||||
### Ignore VMs
|
||||
<img align="left" src="https://cdn.gyptazy.com/images/proxlb-ignore-vm-movement.jpg"/> Guests, such as VMs or CTs, can also be completely ignored. This means, they won't be affected by any migration (even when (anti-)affinity rules are enforced). To ensure a proper resource evaluation, these guests are still collected and evaluated but simply skipped for balancing actions. Another thing is the implementation. While ProxLB might have a very restricted configuration file including the file permissions, this file is only read- and writeable by the Proxmox administrators. However, we might have user and groups who want to define on their own that their systems shouldn't be moved. Therefore, these users can simpy set a specific tag to the guest object - just like the (anti)affinity rules.
|
||||
|
||||
To define a guest to be ignored from the balancing, users assign a tag with the prefix `plb_ignore_$TAG`:
|
||||
|
||||
#### Example for Screenshot
|
||||
```
|
||||
plb_ignore_dev
|
||||
```
|
||||
|
||||
As a result, ProxLB will not migrate this guest with the `plb_ignore_dev` tag to any other node.
|
||||
|
||||
**Note:** Ignored guests are really ignored. Even by enforcing affinity rules this guest will be ignored.
|
||||
|
||||
### Pin VMs to Specific Hypervisor Nodes
|
||||
<img align="left" src="https://cdn.gyptazy.com/images/proxlb-tag-node-pinning.jpg"/> Guests, such as VMs or CTs, can also be pinned to specific (and multiple) nodes in the cluster. This might be usefull when running applications with some special licensing requirements that are only fulfilled on certain nodes. It might also be interesting, when some physical hardware is attached to a node, that is not available in general within the cluster.
|
||||
|
||||
To pin a guest to a specific cluster node, users assign a tag with the prefix `plb_pin_$nodename` to the desired guest:
|
||||
|
||||
#### Example for Screenshot
|
||||
```
|
||||
plb_pin_node03
|
||||
```
|
||||
|
||||
As a result, ProxLB will pin the guest `dev-vm01` to the node `virt03`.
|
||||
|
||||
You can also repeat this step multiple times for different node names to create a potential group of allowed hosts where a the guest may be served on. In this case, ProxLB takes the node with the lowest used resources according to the defined balancing values from this group.
|
||||
|
||||
**Note:** The given node names from the tag are validated. This means, ProxLB validated if the given node name is really part of the cluster. In case of a wrongly defined or unavailable node name it continous to use the regular processes to make sure the guest keeps running.
|
||||
|
||||
## Maintenance
|
||||
<img src="https://cdn.gyptazy.com/images/proxlb-rebalancing-demo.gif"/>
|
||||
|
||||
The `maintenance_nodes` option allows operators to designate one or more Proxmox nodes for maintenance mode. When a node is set to maintenance, no new guest workloads will be assigned to it, and all existing workloads will be migrated to other available nodes within the cluster. This process ensures that (anti)-affinity rules and resource availability are respected, preventing disruptions while maintaining optimal performance across the infrastructure.
|
||||
|
||||
### Adding / Removing Nodes from Maintenance
|
||||
Within the section `proxmox_cluster` you can define the key `maintenance_nodes` as a list object. Simply add/remove one or more nodes with their equal name in the cluster and restart the daemon.
|
||||
```
|
||||
proxmox_cluster:
|
||||
maintenance_nodes: ['virt66.example.com']
|
||||
```
|
||||
Afterwards, all guest objects will be moved to other nodes in the cluster by ensuring the best balancing.
|
||||
|
||||
## Misc
|
||||
### Bugs
|
||||
Bugs can be reported via the GitHub issue tracker [here](https://github.com/gyptazy/ProxLB/issues). You may also report bugs via email or deliver PRs to fix them on your own. Therefore, you might also see the contributing chapter.
|
||||
|
||||
### Contributing
|
||||
Feel free to add further documentation, to adjust already existing one or to contribute with code. Please take care about the style guide and naming conventions. You can find more in our [CONTRIBUTING.md](https://github.com/gyptazy/ProxLB/blob/main/CONTRIBUTING.md) file.
|
||||
|
||||
### Documentation
|
||||
You can also find additional and more detailed documentation within the [docs/](https://github.com/gyptazy/ProxLB/tree/main/docs) directory.
|
||||
|
||||
### Support
|
||||
If you need assistance or have any questions, we offer support through our dedicated [chat room](https://matrix.to/#/#proxlb:gyptazy.com) in Matrix or [Discord](https://discord.gg/JemGu7WbfQ). Join our community for real-time help, advice, and discussions. The Matrix and Discord room are bridged to ensure that the communication is not splitted - so simply feel free to join which fits most to you!
|
||||
|
||||
Connect with us in our dedicated chat room for immediate support and live interaction with other users and developers. You can also visit our [GitHub Community](https://github.com/gyptazy/ProxLB/discussions/) to post your queries, share your experiences, and get support from fellow community members and moderators. You may also just open directly an issue [here](https://github.com/gyptazy/ProxLB/issues) on GitHub.
|
||||
|
||||
| Support Channel | Link |
|
||||
|------|:------:|
|
||||
| Matrix | [#proxlb:gyptazy.com](https://matrix.to/#/#proxlb:gyptazy.com) |
|
||||
| Discord | [Discord](https://discord.gg/JemGu7WbfQ) |
|
||||
| GitHub Community | [GitHub Community](https://github.com/gyptazy/ProxLB/discussions/)
|
||||
| GitHub | [ProxLB GitHub](https://github.com/gyptazy/ProxLB/issues) |
|
||||
|
||||
**Note:** Please always keep in mind that this is a one-man show project without any further help. This includes coding, testing, packaging and all the infrastructure around it to keep this project up and running.
|
||||
|
||||
### Author(s)
|
||||
* Florian Paul Azim Hoberg @gyptazy (https://gyptazy.com)
|
||||
## Reasons
|
||||
You can find more details about this in [my blog post](https://gyptazy.com/blog/proxlb-project-handover-to-credativ/).
|
||||
|
||||
@@ -19,17 +19,63 @@ proxmox_cluster:
|
||||
balancing:
|
||||
enable: True
|
||||
enforce_affinity: False
|
||||
enforce_pinning: False
|
||||
parallel: False
|
||||
# If running parallel job, you can define
|
||||
# the amount of prallel jobs (default: 5)
|
||||
parallel_jobs: 1
|
||||
live: True
|
||||
with_local_disks: True
|
||||
balance_types: ['vm', 'ct']
|
||||
max_job_validation: 1800
|
||||
balanciness: 5
|
||||
method: memory
|
||||
mode: used
|
||||
with_conntrack_state: True
|
||||
balance_types: ['vm', 'ct'] # 'vm' | 'ct'
|
||||
max_job_validation: 1800 # Maximum time (in seconds) a job validation may take
|
||||
memory_threshold: 75 # Optional: Maximum threshold (in percent) to trigger balancing actions
|
||||
balanciness: 5 # Maximum delta of resource usage between highest and lowest usage node
|
||||
method: memory # 'memory' | 'cpu' | 'disk'
|
||||
mode: used # 'assigned' | 'used' | 'psi'
|
||||
balance_larger_guests_first: False # Option to prioritize balancing of larger or smaller guests first
|
||||
node_resource_reserve: # Optional: Define resource reservations for nodes (in GB)
|
||||
defaults: # Default reservation values applying to all nodes (unless explicitly overridden)
|
||||
memory: 4 # Default: 4 GB memory reserved per node
|
||||
node01: # Specific node reservation override for node 'node01'
|
||||
memory: 6 # Specific: 6 GB memory reserved for node 'node01'
|
||||
# # PSI thresholds only apply when using mode 'psi'
|
||||
# psi:
|
||||
# nodes:
|
||||
# memory:
|
||||
# pressure_full: 0.20
|
||||
# pressure_some: 0.20
|
||||
# pressure_spikes: 1.00
|
||||
# cpu:
|
||||
# pressure_full: 0.20
|
||||
# pressure_some: 0.20
|
||||
# pressure_spikes: 1.00
|
||||
# disk:
|
||||
# pressure_full: 0.20
|
||||
# pressure_some: 0.20
|
||||
# pressure_spikes: 1.00
|
||||
# guests:
|
||||
# memory:
|
||||
# pressure_full: 0.20
|
||||
# pressure_some: 0.20
|
||||
# pressure_spikes: 1.00
|
||||
# cpu:
|
||||
# pressure_full: 0.20
|
||||
# pressure_some: 0.20
|
||||
# pressure_spikes: 1.00
|
||||
# disk:
|
||||
# pressure_full: 0.20
|
||||
# pressure_some: 0.20
|
||||
# pressure_spikes: 1.00
|
||||
pools: # Optional: Define affinity/anti-affinity rules per pool
|
||||
dev: # Pool name: dev
|
||||
type: affinity # Type: affinity (keeping VMs together)
|
||||
de-nbg01-db: # Pool name: de-nbg01-db
|
||||
type: anti-affinity # Type: anti-affinity (spreading VMs apart)
|
||||
pin: # Define a pinning og guests to specific node(s)
|
||||
- virt66
|
||||
- virt77
|
||||
strict: False # Disable strict mode of node pinning for this pool
|
||||
|
||||
service:
|
||||
daemon: True
|
||||
|
||||
73
debian/changelog
vendored
73
debian/changelog
vendored
@@ -1,3 +1,76 @@
|
||||
proxlb (1.1.11) stable; urgency=medium
|
||||
|
||||
* Add support for native Proxmox HA/Affinity rules. (Closes: #391)
|
||||
* Add safety guard to avoid node overprovisioning. (Closes: #275)
|
||||
* Fix affinity rules pre-validation (avoid rebalancing if already ensured). (Closes: #335)
|
||||
* Add resource reservation support for PVE nodes. (Closes: #373)
|
||||
* Change/Adjust balancing and sorting behaviour. (Closes: #378)
|
||||
* Add control over balancing workloads by prefering smaller/larger guest objects. (Closes: #387)
|
||||
* Fix pinning of guest and node relations when using pool based pinning. (Closes: #395)
|
||||
* Add validation of HA jobs by fetching the related child jobs. (Closes: #402)
|
||||
* Add support for configuring node-pinning strictness (mode: strict/prefer). (Closes: #406)
|
||||
* Fix that tag based ignored guests got still moved. (Closes: #408)
|
||||
* Add parameter to enforce guest node relationships when pinned even when the cluster is balanced. (Closes: #414)
|
||||
|
||||
-- Florian Paul Azim Hoberg <gyptazy@gyptazy.com> Mon, 12 Jan 2026 11:11:04 +0001
|
||||
|
||||
proxlb (1.1.10) stable; urgency=medium
|
||||
|
||||
* Prevent redundant rebalancing by validating existing affinity enforcement before taking actions. (Closes: #335)
|
||||
* Add safety-guard for PVE 8 users when activating conntrack-aware migrations mistakenly. (Closes: #359)
|
||||
* Fix the Proxmox API connection validation which returned a false-positive logging message of timeouts. (Closes: #361)
|
||||
* Refactored the whole Proxmox API connection function. (Closes: #361)
|
||||
* Fix a crash during PVE resource pool enumeration by skipping members not having a 'name' property. (Closes: #368)
|
||||
|
||||
-- Florian Paul Azim Hoberg <gyptazy@gyptazy.com> Tue, 25 Nov 2025 09:12:04 +0001
|
||||
|
||||
proxlb (1.1.9.1) stable; urgency=medium
|
||||
|
||||
* Fix quoting in f-strings which may cause issues on PVE 8 / Debian Bookworm systems. (Closes: #352)
|
||||
|
||||
-- Florian Paul Azim Hoberg <gyptazy@gyptazy.com> Thu, 30 Oct 2025 17:41:02 +0001
|
||||
|
||||
proxlb (1.1.9) stable; urgency=medium
|
||||
|
||||
* Add pressure (PSI) based balancing for memory, cpu, disk (req. PVE9 or greater). (Closes: #339)
|
||||
* Add (memory) threshold for nodes before running balancing. (Closes: #342)
|
||||
* Add affinity/anti-affinity support by pools. (Closes: #343)
|
||||
|
||||
-- Florian Paul Azim Hoberg <gyptazy@gyptazy.com> Thu, 30 Oct 2025 06:58:43 +0001
|
||||
|
||||
proxlb (1.1.8) stable; urgency=medium
|
||||
|
||||
* Fix API errors when using conntrack aware migration with older PVE version. (Closes: #318)
|
||||
* Add a static ProxLB prefix to the log output when used by journal handler. (Closes: #329)
|
||||
|
||||
-- Florian Paul Azim Hoberg <gyptazy@gyptazy.com> Thu, 09 Oct 2025 09:04:13 +0002
|
||||
|
||||
proxlb (1.1.7) stable; urgency=medium
|
||||
|
||||
* Add conntrack state aware migrations of VMs. (Closes: #305)
|
||||
* Add graceful shutdown for SIGINT command. (Closes: #304)
|
||||
* Fix crash when validating absent migration job ids. (Closes: #308)
|
||||
* Fix guest object names are not being evaluated in debug log. (Closes: #310)
|
||||
* Note: Have a great Dutch Proxmox Day 2025!
|
||||
|
||||
-- Florian Paul Azim Hoberg <gyptazy@gyptazy.com> Thu, 04 Sep 2025 19:23:51 +0000
|
||||
|
||||
proxlb (1.1.6.1) stable; urgency=medium
|
||||
|
||||
* Validate for node presence when pinning VMs to avoid crashing. (Closes: #296)
|
||||
|
||||
-- Florian Paul Azim Hoberg <gyptazy@gyptazy.com> Thu, 04 Sep 2025 19:23:51 +0000
|
||||
|
||||
proxlb (1.1.6) stable; urgency=medium
|
||||
|
||||
* Add validation for provided API user token id to avoid confusions. (Closes: #291)
|
||||
* Fix stacktrace output when validating permissions on non existing users in Proxmox. (Closes: #291)
|
||||
* Fix Overprovisioning first node if anti_affinity_group has only one member. (Closes: #295)
|
||||
* Validate for node presence when pinning guests to avoid crashing. (Closes: #296)
|
||||
* Fix balancing evaluation of guest types (e.g., VM or CT). (Closes: #268)
|
||||
|
||||
-- Florian Paul Azim Hoberg <gyptazy@gyptazy.com> Thu, 04 Sep 2025 05:12:19 +0000
|
||||
|
||||
proxlb (1.1.5) stable; urgency=medium
|
||||
|
||||
* Allow custom API ports instead of fixed tcp/8006. (Closes: #260)
|
||||
|
||||
6
debian/control
vendored
6
debian/control
vendored
@@ -7,6 +7,6 @@ Build-Depends: debhelper-compat (= 13), dh-python, python3-all, python3-setuptoo
|
||||
|
||||
Package: proxlb
|
||||
Architecture: all
|
||||
Depends: ${python3:Depends}, ${misc:Depends}, python3-requests, python3-urllib3, python3-proxmoxer, python3-yaml
|
||||
Description: A DRS alike Load Balancer for Proxmox Clusters
|
||||
An advanced DRS alike loadbalancer for Proxmox clusters that also supports maintenance modes and affinity/anti-affinity rules.
|
||||
Depends: ${python3:Depends}, ${misc:Depends}, python3-requests, python3-urllib3, python3-packaging, python3-proxmoxer, python3-yaml
|
||||
Description: An advanced resource scheduler and load balancer for Proxmox clusters
|
||||
An advanced resource scheduler and load balancer for Proxmox clusters that also supports maintenance mode and affinity/anti-affinity rules.
|
||||
|
||||
@@ -55,7 +55,7 @@ ProxLB itself requires minimal system resources to operate. However, for managin
|
||||
|
||||
|
||||
## Where To Run?
|
||||
ProxLB can run on pretty anthing and only requires you to have a network connectivity to any of the Proxmox host's API (usually on tcp/8006).
|
||||
ProxLB is lightweight and flexible where it runs on nearly any environment and only needs access to your Proxmox host’s API endpoint (commonly TCP port 8006).
|
||||
|
||||
Therefore, you can simply run ProxLB on:
|
||||
* Bare-metal Systems
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
- [Quick-Start](#quick-start)
|
||||
- [Details](#details)
|
||||
- [Debian Packages (.deb files)](#debian-packages-deb-files)
|
||||
- [Repo Mirror and Proxmox Offline Mirror Support](#repo-mirror-and-proxmox-offline-mirror-support)
|
||||
- [RedHat Package](#redhat-package)
|
||||
- [Container Images / Docker](#container-images--docker)
|
||||
- [Overview of Images](#overview-of-images)
|
||||
@@ -83,6 +84,27 @@ vi /etc/proxlb/proxlb.yaml
|
||||
systemctl start proxlb
|
||||
```
|
||||
|
||||
#### Repo Mirror and Proxmox Offline Mirror Support
|
||||
ProxLB uses the supported flat mirror style for the Debian repository. Unfortunately, not all offline-mirror applications support it. One of the known ones is the official *proxmox-offline-mirror* which is unable to handle flat repositories (see also: [#385](https://github.com/gyptazy/ProxLB/issues/385)).
|
||||
|
||||
Therefore, we currently operate and support both ways to avoid everyone force switching to the new repository. As a result, you can simply use this repository:
|
||||
```
|
||||
deb https://repo.gyptazy.com/proxlb stable main
|
||||
```
|
||||
|
||||
**Example Config for proxmox-offline-mirror:**
|
||||
|
||||
An example config for the proxmox-offline-mirror would look like:
|
||||
```
|
||||
mirror: proxlb
|
||||
architectures amd64
|
||||
base-dir /var/lib/proxmox-offline-mirror/mirrors/
|
||||
key-path /etc/apt/trusted.gpg.d/proxlb.asc
|
||||
repository deb https://repo.gyptazy.com/proxlb stable main
|
||||
sync true
|
||||
verify true
|
||||
```
|
||||
|
||||
### RedHat Package
|
||||
There's currently no official support for RedHat based systems. However, there's a dummy .rpm package for such systems in the pipeline which can be found here:
|
||||
* https://github.com/gyptazy/ProxLB/actions/workflows/20-pipeline-build-rpm-package.yml
|
||||
|
||||
@@ -20,6 +20,10 @@
|
||||
7. [Run as a Systemd-Service](#run-as-a-systemd-service)
|
||||
8. [SSL Self-Signed Certificates](#ssl-self-signed-certificates)
|
||||
9. [Node Maintenances](#node-maintenances)
|
||||
10. [Balancing Methods](#balancing-methods)
|
||||
1. [Used Resources](#used-resources)
|
||||
2. [Assigned Resources](#assigned-resources)
|
||||
3. [Pressure (PSI) based Resources](#pressure-psi-based-resources)
|
||||
|
||||
## Authentication / User Accounts / Permissions
|
||||
### Authentication
|
||||
@@ -76,8 +80,8 @@ ProxLB provides an advanced mechanism to define affinity and anti-affinity rules
|
||||
|
||||
ProxLB implements affinity and anti-affinity rules through a tag-based system within the Proxmox web interface. Each guest (virtual machine or container) can be assigned specific tags, which then dictate its placement behavior. This method maintains a streamlined and secure approach to managing VM relationships while preserving Proxmox’s inherent permission model.
|
||||
|
||||
#### Affinity Rules
|
||||
<img align="left" src="https://cdn.gyptazy.com/images/proxlb-affinity-rules.jpg"/> Affinity rules are used to group certain VMs together, ensuring that they run on the same host whenever possible. This can be beneficial for workloads requiring low-latency communication, such as clustered databases or application servers that frequently exchange data.
|
||||
#### Affinity Rules by Tags
|
||||
<img align="left" src="https://cdn.gyptazy.com/img/proxlb-affinity-rules.jpg"/> Affinity rules are used to group certain VMs together, ensuring that they run on the same host whenever possible. This can be beneficial for workloads requiring low-latency communication, such as clustered databases or application servers that frequently exchange data.
|
||||
|
||||
To define an affinity rule which keeps all guests assigned to this tag together on a node, users assign a tag with the prefix `plb_affinity_$TAG`:
|
||||
|
||||
@@ -88,8 +92,20 @@ plb_affinity_talos
|
||||
|
||||
As a result, ProxLB will attempt to place all VMs with the `plb_affinity_web` tag on the same host (see also the attached screenshot with the same node).
|
||||
|
||||
#### Anti-Affinity Rules
|
||||
<img align="left" src="https://cdn.gyptazy.com/images/proxlb-anti-affinity-rules.jpg"/> Conversely, anti-affinity rules ensure that designated VMs do not run on the same physical host. This is particularly useful for high-availability setups, where redundancy is crucial. Ensuring that critical services are distributed across multiple hosts reduces the risk of a single point of failure.
|
||||
#### Affinity Rules by Pools
|
||||
Antoher approach is by using pools in Proxmox. This way, it can easily also combined with other resources like backup jobs. However, in this approach you need to modify the ProxLB config file to your needs. Within the `balancing` section you can create a dict of pools, including the pool name and the affinity type. Please see the example for further details:
|
||||
|
||||
**Example Config**
|
||||
```
|
||||
balancing:
|
||||
[...]
|
||||
pools: # Optional: Define affinity/anti-affinity rules per pool
|
||||
dev: # Pool name: dev
|
||||
type: affinity # Type: affinity (keeping VMs together)
|
||||
```
|
||||
|
||||
#### Anti-Affinity Rules by Tags
|
||||
<img align="left" src="https://cdn.gyptazy.com/img/proxlb-anti-affinity-rules.jpg"/> Conversely, anti-affinity rules ensure that designated VMs do not run on the same physical host. This is particularly useful for high-availability setups, where redundancy is crucial. Ensuring that critical services are distributed across multiple hosts reduces the risk of a single point of failure.
|
||||
|
||||
To define an anti-affinity rule that ensures to not move systems within this group to the same node, users assign a tag with the prefix:
|
||||
|
||||
@@ -102,6 +118,18 @@ As a result, ProxLB will try to place the VMs with the `plb_anti_affinity_ntp` t
|
||||
|
||||
**Note:** While this ensures that ProxLB tries distribute these VMs across different physical hosts within the Proxmox cluster this may not always work. If you have more guests attached to the group than nodes in the cluster, we still need to run them anywhere. If this case occurs, the next one with the most free resources will be selected.
|
||||
|
||||
#### Anti-Affinity Rules by Pools
|
||||
Antoher approach is by using pools in Proxmox. This way, it can easily also combined with other resources like backup jobs. However, in this approach you need to modify the ProxLB config file to your needs. Within the `balancing` section you can create a dict of pools, including the pool name and the affinity type. Please see the example for further details:
|
||||
|
||||
**Example Config**
|
||||
```
|
||||
balancing:
|
||||
[...]
|
||||
pools: # Optional: Define affinity/anti-affinity rules per pool
|
||||
de-nbg01-db: # Pool name: de-nbg01-db
|
||||
type: anti-affinity # Type: anti-affinity (spreading VMs apart)
|
||||
````
|
||||
|
||||
### Affinity / Anti-Affinity Enforcing
|
||||
When a cluster is already balanced and does not require further adjustments, enabling the enforce_affinity parameter ensures that affinity and anti-affinity rules are still respected. This parameter prioritizes the placement of guest objects according to these rules, even if it leads to slight resource imbalances or increased migration overhead. Regularly reviewing and updating these rules, along with monitoring cluster performance, helps maintain optimal performance and reliability. By carefully managing these aspects, you can create a cluster environment that meets your specific needs and maintains a good balance of resources.
|
||||
|
||||
@@ -113,7 +141,7 @@ balancing:
|
||||
*Note: This may have impacts to the cluster. Depending on the created group matrix, the result may also be an unbalanced cluster.*
|
||||
|
||||
### Ignore VMs / CTs
|
||||
<img align="left" src="https://cdn.gyptazy.com/images/proxlb-ignore-vm-movement.jpg"/> Guests, such as VMs or CTs, can also be completely ignored. This means, they won't be affected by any migration (even when (anti-)affinity rules are enforced). To ensure a proper resource evaluation, these guests are still collected and evaluated but simply skipped for balancing actions. Another thing is the implementation. While ProxLB might have a very restricted configuration file including the file permissions, this file is only read- and writeable by the Proxmox administrators. However, we might have user and groups who want to define on their own that their systems shouldn't be moved. Therefore, these users can simpy set a specific tag to the guest object - just like the (anti)affinity rules.
|
||||
<img align="left" src="https://cdn.gyptazy.com/img/proxlb-ignore-vm-movement.jpg"/> Guests, such as VMs or CTs, can also be completely ignored. This means, they won't be affected by any migration (even when (anti-)affinity rules are enforced). To ensure a proper resource evaluation, these guests are still collected and evaluated but simply skipped for balancing actions. Another thing is the implementation. While ProxLB might have a very restricted configuration file including the file permissions, this file is only read- and writeable by the Proxmox administrators. However, we might have user and groups who want to define on their own that their systems shouldn't be moved. Therefore, these users can simpy set a specific tag to the guest object - just like the (anti)affinity rules.
|
||||
|
||||
To define a guest to be ignored from the balancing, users assign a tag with the prefix `plb_ignore_$TAG`:
|
||||
|
||||
@@ -235,4 +263,115 @@ The maintenance_nodes key must be defined as a list, even if it only includes a
|
||||
* No new workloads will be balanced or migrated onto it.
|
||||
* Any existing workloads currently running on the node will be migrated away in accordance with the configured balancing strategies, assuming resources on other nodes allow.
|
||||
|
||||
This feature is particularly useful during planned maintenance, upgrades, or troubleshooting, ensuring that services continue to run with minimal disruption while the specified node is being worked on.
|
||||
This feature is particularly useful during planned maintenance, upgrades, or troubleshooting, ensuring that services continue to run with minimal disruption while the specified node is being worked on.
|
||||
|
||||
## 10. Balancing Methods
|
||||
ProxLB provides multiple balancing modes that define *how* resources are evaluated and compared during cluster balancing.
|
||||
Each mode reflects a different strategy for determining load and distributing guests (VMs or containers) between nodes.
|
||||
|
||||
Depending on your environment, provisioning strategy, and performance goals, you can choose between:
|
||||
|
||||
| Mode | Description | Typical Use Case |
|
||||
|------|--------------|------------------|
|
||||
| `used` | Uses the *actual runtime resource usage* (e.g. CPU, memory, disk). | Dynamic or lab environments with frequent workload changes and tolerance for overprovisioning. |
|
||||
| `assigned` | Uses the *statically defined resource allocations* from guest configurations. | Production or SLA-driven clusters that require guaranteed resources and predictable performance. |
|
||||
| `psi` | Uses Linux *Pressure Stall Information (PSI)* metrics to evaluate real system contention and pressure. | Advanced clusters that require pressure-aware decisions for proactive rebalancing. |
|
||||
|
||||
### 10.1 Used Resources
|
||||
When **mode: `used`** is configured, ProxLB evaluates the *real usage metrics* of guest objects (VMs and CTs).
|
||||
It collects the current CPU, memory, and disk usage directly from the Proxmox API to determine the *actual consumption* of each guest and node.
|
||||
|
||||
This mode is ideal for **dynamic environments** where workloads frequently change and **overprovisioning is acceptable**. It provides the most reactive balancing behavior, since decisions are based on live usage instead of static assignment.
|
||||
|
||||
Typical scenarios include:
|
||||
- Production environments to distribute workloads across the nodes.
|
||||
- Test or development clusters with frequent VM changes.
|
||||
- Clusters where resource spikes are short-lived.
|
||||
- Environments where slight resource contention is tolerable.
|
||||
|
||||
#### Example Configuration
|
||||
```yaml
|
||||
balancing:
|
||||
mode: used
|
||||
```
|
||||
|
||||
### 10.2 Assigned Resources
|
||||
When **mode: `assigned`** is configured, ProxLB evaluates the *provisioned or allocated resources* of each guest (VM or CT) instead of their runtime usage.
|
||||
It uses data such as **CPU cores**, **memory limits**, and **disk allocations** defined in Proxmox to calculate how much of each node’s capacity is reserved.
|
||||
|
||||
This mode is ideal for **production clusters** where:
|
||||
- Overcommitment is *not allowed or only minimally tolerated*.
|
||||
- Each node’s workload is planned based on the assigned capacities.
|
||||
- Administrators want predictable resource distribution aligned with provisioning policies.
|
||||
|
||||
Unlike the `used` mode, `assigned` focuses purely on the *declared configuration* of guests and remains stable even if actual usage varies temporarily.
|
||||
|
||||
Typical scenarios include:
|
||||
- Enterprise environments with SLA or QoS requirements.
|
||||
- Clusters where workloads are sized deterministically.
|
||||
- Situations where consistent node utilization and capacity awareness are crucial.
|
||||
|
||||
#### Example Configuration
|
||||
```yaml
|
||||
balancing:
|
||||
mode: assigned
|
||||
```
|
||||
|
||||
### 10.3 Pressure (PSI) based Resources
|
||||
> [!IMPORTANT]
|
||||
> PSI based balancing is still in beta! If you find any bugs, please raise an issue including metrics of all nodes and affected guests. You can provide metrics directly from PVE or Grafana (via node_exporter or pve_exporter).
|
||||
|
||||
When **mode: `psi`** is configured, ProxLB uses the **Linux Pressure Stall Information (PSI)** interface to measure the *real-time pressure* on system resources such as **CPU**, **memory**, and **disk I/O**.
|
||||
Unlike the `used` or `assigned` modes, which rely on static or average metrics, PSI provides *direct insight into how often and how long tasks are stalled* because of insufficient resources.
|
||||
|
||||
This enables ProxLB to make **proactive balancing decisions** — moving workloads *before* performance degradation becomes visible to the user.
|
||||
|
||||
**IMPORTANT**: Predicting distributing workloads is dangerous and might not result into the expected state. Therefore, ProxLB migrates only a single instance each 60 minutes to obtain new real-metrics and to validate if further changes are required. Keep in mind, that migrations are also costly and should be avoided as much as possible.
|
||||
|
||||
PSI metrics are available for both **nodes** and **guest objects**, allowing fine-grained balancing decisions:
|
||||
- **Node-level PSI:** Detects cluster nodes under systemic load or contention.
|
||||
- **Guest-level PSI:** Identifies individual guests suffering from memory, CPU, or I/O stalls.
|
||||
|
||||
### PSI Metrics Explained
|
||||
Each monitored resource defines three pressure thresholds:
|
||||
| Key | Description |
|
||||
|-----|--------------|
|
||||
| `pressure_some` | Indicates partial stall conditions where some tasks are waiting for a resource. |
|
||||
| `pressure_full` | Represents complete stall conditions where *all* tasks are blocked waiting for a resource. |
|
||||
| `pressure_spikes` | Defines short-term burst conditions that may signal saturation spikes. |
|
||||
|
||||
These thresholds are expressed in **percentages** and represent how much time the kernel reports stalls over specific averaging windows (e.g. 5s, 10s, 60s).
|
||||
|
||||
### Example Configuration
|
||||
|
||||
```yaml
|
||||
balancing:
|
||||
mode: psi
|
||||
psi:
|
||||
nodes:
|
||||
memory:
|
||||
pressure_full: 0.20
|
||||
pressure_some: 0.20
|
||||
pressure_spikes: 1.00
|
||||
cpu:
|
||||
pressure_full: 0.20
|
||||
pressure_some: 0.20
|
||||
pressure_spikes: 1.00
|
||||
disk:
|
||||
pressure_full: 0.20
|
||||
pressure_some: 0.20
|
||||
pressure_spikes: 1.00
|
||||
guests:
|
||||
memory:
|
||||
pressure_full: 0.20
|
||||
pressure_some: 0.20
|
||||
pressure_spikes: 1.00
|
||||
cpu:
|
||||
pressure_full: 0.20
|
||||
pressure_some: 0.20
|
||||
pressure_spikes: 1.00
|
||||
disk:
|
||||
pressure_full: 0.20
|
||||
pressure_some: 0.20
|
||||
pressure_spikes: 1.00
|
||||
```
|
||||
|
||||
@@ -1,24 +1,6 @@
|
||||
apiVersion: v3
|
||||
apiVersion: v2
|
||||
name: proxlb
|
||||
description: A Helm chart for self-hosted ProxLB
|
||||
|
||||
# A chart can be either an 'application' or a 'library' chart.
|
||||
#
|
||||
# Application charts are a collection of templates that can be packaged into versioned archives
|
||||
# to be deployed.
|
||||
#
|
||||
# Library charts provide useful utilities or functions for the chart developer. They're included as
|
||||
# a dependency of application charts to inject those utilities and functions into the rendering
|
||||
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
|
||||
type: application
|
||||
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: "1.1.5"
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
# follow Semantic Versioning. They should reflect the version the application is using.
|
||||
# It is recommended to use it with quotes.
|
||||
appVersion: "v1.1.5"
|
||||
version: "1.1.11"
|
||||
appVersion: "v1.1.11"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
image:
|
||||
registry: cr.gyptazy.com
|
||||
repository: proxlb/proxlb
|
||||
tag: v1.1.5
|
||||
tag: v1.1.11
|
||||
pullPolicy: IfNotPresent
|
||||
imagePullSecrets: [ ]
|
||||
|
||||
@@ -43,6 +43,7 @@ configmap:
|
||||
parallel_jobs: 1
|
||||
live: True
|
||||
with_local_disks: True
|
||||
with_conntrack_state: True
|
||||
balance_types: [ 'vm', 'ct' ]
|
||||
max_job_validation: 1800
|
||||
balanciness: 5
|
||||
@@ -57,4 +58,4 @@ configmap:
|
||||
enable: False
|
||||
time: 1
|
||||
format: "hours"
|
||||
log_level: INFO
|
||||
log_level: INFO
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
VERSION="1.1.4"
|
||||
VERSION="1.1.9.1"
|
||||
|
||||
# ProxLB
|
||||
sed -i "s/^__version__ = .*/__version__ = \"$VERSION\"/" "proxlb/utils/version.py"
|
||||
@@ -8,5 +8,6 @@ sed -i "s/version=\"[0-9]*\.[0-9]*\.[0-9]*\"/version=\"$VERSION\"/" setup.py
|
||||
# Helm Chart
|
||||
sed -i "s/^version: .*/version: \"$VERSION\"/" helm/proxlb/Chart.yaml
|
||||
sed -i "s/^appVersion: .*/appVersion: \"v$VERSION\"/" helm/proxlb/Chart.yaml
|
||||
sed -i "s/^tag: .*/tag: \"v$VERSION\"/" helm/proxlb/values.yaml
|
||||
|
||||
echo "OK: Versions have been sucessfully set to $VERSION"
|
||||
|
||||
@@ -19,10 +19,13 @@ from utils.cli_parser import CliParser
|
||||
from utils.config_parser import ConfigParser
|
||||
from utils.proxmox_api import ProxmoxApi
|
||||
from models.nodes import Nodes
|
||||
from models.features import Features
|
||||
from models.guests import Guests
|
||||
from models.groups import Groups
|
||||
from models.calculations import Calculations
|
||||
from models.balancing import Balancing
|
||||
from models.pools import Pools
|
||||
from models.ha_rules import HaRules
|
||||
from utils.helper import Helper
|
||||
|
||||
|
||||
@@ -33,8 +36,9 @@ def main():
|
||||
# Initialize logging handler
|
||||
logger = SystemdLogger(level=logging.INFO)
|
||||
|
||||
# Signal handler for SIGHUP
|
||||
# Initialize handlers
|
||||
signal.signal(signal.SIGHUP, Helper.handler_sighup)
|
||||
signal.signal(signal.SIGINT, Helper.handler_sigint)
|
||||
|
||||
# Parses arguments passed from the CLI
|
||||
cli_parser = CliParser()
|
||||
@@ -70,17 +74,27 @@ def main():
|
||||
# Get all required objects from the Proxmox cluster
|
||||
meta = {"meta": proxlb_config}
|
||||
nodes = Nodes.get_nodes(proxmox_api, proxlb_config)
|
||||
guests = Guests.get_guests(proxmox_api, nodes, meta)
|
||||
meta = Features.validate_any_non_pve9_node(meta, nodes)
|
||||
pools = Pools.get_pools(proxmox_api)
|
||||
ha_rules = HaRules.get_ha_rules(proxmox_api, meta)
|
||||
guests = Guests.get_guests(proxmox_api, pools, ha_rules, nodes, meta, proxlb_config)
|
||||
groups = Groups.get_groups(guests, nodes)
|
||||
|
||||
# Merge obtained objects from the Proxmox cluster for further usage
|
||||
proxlb_data = {**meta, **nodes, **guests, **groups}
|
||||
proxlb_data = {**meta, **nodes, **guests, **pools, **ha_rules, **groups}
|
||||
Helper.log_node_metrics(proxlb_data)
|
||||
|
||||
# Validate usable features by PVE versions
|
||||
Features.validate_available_features(proxlb_data)
|
||||
|
||||
# Update the initial node resource assignments
|
||||
# by the previously created groups.
|
||||
Calculations.set_node_assignments(proxlb_data)
|
||||
Helper.log_node_metrics(proxlb_data, init=False)
|
||||
Calculations.set_node_hot(proxlb_data)
|
||||
Calculations.set_guest_hot(proxlb_data)
|
||||
Calculations.get_most_free_node(proxlb_data, cli_args.best_node)
|
||||
Calculations.validate_affinity_map(proxlb_data)
|
||||
Calculations.relocate_guests_on_maintenance_nodes(proxlb_data)
|
||||
Calculations.get_balanciness(proxlb_data)
|
||||
Calculations.relocate_guests(proxlb_data)
|
||||
|
||||
@@ -91,7 +91,7 @@ class Balancing:
|
||||
# VM Balancing
|
||||
if guest_meta["type"] == "vm":
|
||||
if 'vm' in proxlb_data["meta"]["balancing"].get("balance_types", []):
|
||||
logger.debug("Balancing: Balancing for guest {guest_name} of type VM started.")
|
||||
logger.debug(f"Balancing: Balancing for guest {guest_name} of type VM started.")
|
||||
job_id = self.exec_rebalancing_vm(proxmox_api, proxlb_data, guest_name)
|
||||
else:
|
||||
logger.debug(
|
||||
@@ -101,7 +101,7 @@ class Balancing:
|
||||
# CT Balancing
|
||||
elif guest_meta["type"] == "ct":
|
||||
if 'ct' in proxlb_data["meta"]["balancing"].get("balance_types", []):
|
||||
logger.debug("Balancing: Balancing for guest {guest_name} of type CT started.")
|
||||
logger.debug(f"Balancing: Balancing for guest {guest_name} of type CT started.")
|
||||
job_id = self.exec_rebalancing_ct(proxmox_api, proxlb_data, guest_name)
|
||||
else:
|
||||
logger.debug(
|
||||
@@ -122,7 +122,8 @@ class Balancing:
|
||||
|
||||
# Wait for all jobs in the current chunk to complete
|
||||
for guest_name, node, job_id in jobs_to_wait:
|
||||
self.get_rebalancing_job_status(proxmox_api, proxlb_data, guest_name, node, job_id)
|
||||
if job_id:
|
||||
self.get_rebalancing_job_status(proxmox_api, proxlb_data, guest_name, node, job_id)
|
||||
|
||||
def exec_rebalancing_vm(self, proxmox_api: any, proxlb_data: Dict[str, Any], guest_name: str) -> None:
|
||||
"""
|
||||
@@ -143,6 +144,7 @@ class Balancing:
|
||||
guest_id = proxlb_data["guests"][guest_name]["id"]
|
||||
guest_node_current = proxlb_data["guests"][guest_name]["node_current"]
|
||||
guest_node_target = proxlb_data["guests"][guest_name]["node_target"]
|
||||
job_id = None
|
||||
|
||||
if proxlb_data["meta"]["balancing"].get("live", True):
|
||||
online_migration = 1
|
||||
@@ -155,17 +157,23 @@ class Balancing:
|
||||
with_local_disks = 0
|
||||
|
||||
migration_options = {
|
||||
'target': {guest_node_target},
|
||||
'target': guest_node_target,
|
||||
'online': online_migration,
|
||||
'with-local-disks': with_local_disks
|
||||
'with-local-disks': with_local_disks,
|
||||
}
|
||||
|
||||
# Conntrack state aware migrations are not supported in older
|
||||
# PVE versions, so we should not add it by default.
|
||||
if proxlb_data["meta"]["balancing"].get("with_conntrack_state", True):
|
||||
migration_options['with-conntrack-state'] = 1
|
||||
|
||||
try:
|
||||
logger.info(f"Balancing: Starting to migrate VM guest {guest_name} from {guest_node_current} to {guest_node_target}.")
|
||||
job_id = proxmox_api.nodes(guest_node_current).qemu(guest_id).migrate().post(**migration_options)
|
||||
except proxmoxer.core.ResourceException as proxmox_api_error:
|
||||
logger.critical(f"Balancing: Failed to migrate guest {guest_name} of type VM due to some Proxmox errors. Please check if resource is locked or similar.")
|
||||
logger.debug(f"Balancing: Failed to migrate guest {guest_name} of type VM due to some Proxmox errors: {proxmox_api_error}")
|
||||
|
||||
logger.debug("Finished: exec_rebalancing_vm.")
|
||||
return job_id
|
||||
|
||||
@@ -188,6 +196,7 @@ class Balancing:
|
||||
guest_id = proxlb_data["guests"][guest_name]["id"]
|
||||
guest_node_current = proxlb_data["guests"][guest_name]["node_current"]
|
||||
guest_node_target = proxlb_data["guests"][guest_name]["node_target"]
|
||||
job_id = None
|
||||
|
||||
try:
|
||||
logger.info(f"Balancing: Starting to migrate CT guest {guest_name} from {guest_node_current} to {guest_node_target}.")
|
||||
@@ -195,6 +204,7 @@ class Balancing:
|
||||
except proxmoxer.core.ResourceException as proxmox_api_error:
|
||||
logger.critical(f"Balancing: Failed to migrate guest {guest_name} of type CT due to some Proxmox errors. Please check if resource is locked or similar.")
|
||||
logger.debug(f"Balancing: Failed to migrate guest {guest_name} of type CT due to some Proxmox errors: {proxmox_api_error}")
|
||||
|
||||
logger.debug("Finished: exec_rebalancing_ct.")
|
||||
return job_id
|
||||
|
||||
@@ -216,8 +226,23 @@ class Balancing:
|
||||
logger.debug("Starting: get_rebalancing_job_status.")
|
||||
job = proxmox_api.nodes(guest_current_node).tasks(job_id).status().get()
|
||||
|
||||
# Fetch actual migration job status if this got spawned by a HA job
|
||||
if job["type"] == "hamigrate":
|
||||
logger.debug(f"Balancing: Job ID {job_id} (guest: {guest_name}) is a HA migration job. Fetching underlying migration job...")
|
||||
time.sleep(1)
|
||||
vm_id = int(job["id"])
|
||||
qm_migrate_jobs = proxmox_api.nodes(guest_current_node).tasks.get(typefilter="qmigrate", vmid=vm_id, start=0, source="active", limit=1)
|
||||
|
||||
if len(qm_migrate_jobs) > 0:
|
||||
job = qm_migrate_jobs[0]
|
||||
job_id = job["upid"]
|
||||
logger.debug(f'Overwriting job polling for: ID {job_id} (guest: {guest_name}) by {job}')
|
||||
else:
|
||||
logger.debug(f"Balancing: Job ID {job_id} (guest: {guest_name}) is a standard migration job. Proceeding with status check.")
|
||||
|
||||
# Watch job id until it finalizes
|
||||
if job["status"] == "running":
|
||||
# Note: Unsaved jobs are delivered in uppercase from Proxmox API
|
||||
if job.get("status", "").lower() == "running":
|
||||
# Do not hammer the API while
|
||||
# watching the job status
|
||||
time.sleep(10)
|
||||
|
||||
@@ -80,7 +80,7 @@ class Calculations:
|
||||
|
||||
for guest_name in group_meta["guests"]:
|
||||
guest_node_current = proxlb_data["guests"][guest_name]["node_current"]
|
||||
# Update Hardware assignments
|
||||
# Update resource assignments
|
||||
# Update assigned values for the current node
|
||||
logger.debug(f"set_node_assignment of guest {guest_name} on node {guest_node_current} with cpu_total: {proxlb_data['guests'][guest_name]['cpu_total']}, memory_total: {proxlb_data['guests'][guest_name]['memory_total']}, disk_total: {proxlb_data['guests'][guest_name]['disk_total']}.")
|
||||
proxlb_data["nodes"][guest_node_current]["cpu_assigned"] += proxlb_data["guests"][guest_name]["cpu_total"]
|
||||
@@ -93,6 +93,83 @@ class Calculations:
|
||||
|
||||
logger.debug("Finished: set_node_assignments.")
|
||||
|
||||
def set_node_hot(proxlb_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Evaluates node 'full' pressure metrics for memory, cpu, and io
|
||||
against defined thresholds and sets <metric>_pressure_hot = True
|
||||
when a node is considered HOT.
|
||||
|
||||
Returns the modified proxlb_data dict.
|
||||
"""
|
||||
logger.debug("Starting: set_node_hot.")
|
||||
balancing_cfg = proxlb_data.get("meta", {}).get("balancing", {})
|
||||
thresholds = balancing_cfg.get("psi_thresholds", balancing_cfg.get("psi", {}).get("nodes", {}))
|
||||
nodes = proxlb_data.get("nodes", {})
|
||||
|
||||
for node_name, node in nodes.items():
|
||||
|
||||
if node.get("maintenance"):
|
||||
continue
|
||||
|
||||
if node.get("ignore"):
|
||||
continue
|
||||
|
||||
# PSI metrics are only availavble on Proxmox VE 9.0 and higher.
|
||||
if proxlb_data["meta"]["balancing"].get("mode", "used") == "psi":
|
||||
|
||||
if tuple(map(int, proxlb_data["nodes"][node["name"]]["pve_version"].split('.'))) < tuple(map(int, "9.0".split('.'))):
|
||||
logger.critical(f"Proxmox node {node['name']} runs Proxmox VE version {proxlb_data['nodes'][node['name']]['pve_version']}."
|
||||
" PSI metrics require Proxmox VE 9.0 or higher. Balancing deactivated!")
|
||||
|
||||
for metric, threshold in thresholds.items():
|
||||
pressure_full = node.get(f"{metric}_pressure_full_percent", 0.0)
|
||||
pressure_some = node.get(f"{metric}_pressure_some_percent", 0.0)
|
||||
pressure_spikes = node.get(f"{metric}_pressure_full_spikes_percent", 0.0)
|
||||
is_hot = (pressure_full >= threshold["pressure_full"] and pressure_some >= threshold["pressure_some"]) or (pressure_spikes >= threshold["pressure_spikes"])
|
||||
|
||||
if is_hot:
|
||||
logger.debug(f"Set node {node['name']} as hot based on {metric} pressure metrics.")
|
||||
proxlb_data["nodes"][node["name"]][f"{metric}_pressure_hot"] = True
|
||||
proxlb_data["nodes"][node["name"]][f"pressure_hot"] = True
|
||||
else:
|
||||
logger.debug(f"Node {node['name']} is not hot based on {metric} pressure metrics.")
|
||||
|
||||
logger.debug("Finished: set_node_hot.")
|
||||
return proxlb_data
|
||||
|
||||
def set_guest_hot(proxlb_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Evaluates guest 'full' pressure metrics for memory, cpu, and io
|
||||
against defined thresholds and sets <metric>_pressure_hot = True
|
||||
when a guest is considered HOT.
|
||||
|
||||
Returns the modified proxlb_data dict.
|
||||
"""
|
||||
logger.debug("Starting: set_guest_hot.")
|
||||
balancing_cfg = proxlb_data.get("meta", {}).get("balancing", {})
|
||||
thresholds = balancing_cfg.get("psi_thresholds", balancing_cfg.get("psi", {}).get("guests", {}))
|
||||
guests = proxlb_data.get("guests", {})
|
||||
|
||||
for guest_name, guest in guests.items():
|
||||
if guest.get("ignore"):
|
||||
continue
|
||||
|
||||
for metric, threshold in thresholds.items():
|
||||
pressure_full = guest.get(f"{metric}_pressure_full_percent", 0.0)
|
||||
pressure_some = guest.get(f"{metric}_pressure_some_percent", 0.0)
|
||||
pressure_spikes = guest.get(f"{metric}_pressure_full_spikes_percent", 0.0)
|
||||
is_hot = (pressure_full >= threshold["pressure_full"] and pressure_some >= threshold["pressure_some"]) or (pressure_spikes >= threshold["pressure_spikes"])
|
||||
|
||||
if is_hot:
|
||||
logger.debug(f"Set guest {guest['name']} as hot based on {metric} pressure metrics.")
|
||||
proxlb_data["guests"][guest["name"]][f"{metric}_pressure_hot"] = True
|
||||
proxlb_data["guests"][guest["name"]][f"pressure_hot"] = True
|
||||
else:
|
||||
logger.debug(f"guest {guest['name']} is not hot based on {metric} pressure metrics.")
|
||||
|
||||
logger.debug("Finished: set_guest_hot.")
|
||||
return proxlb_data
|
||||
|
||||
@staticmethod
|
||||
def get_balanciness(proxlb_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
@@ -113,7 +190,66 @@ class Calculations:
|
||||
method = proxlb_data["meta"]["balancing"].get("method", "memory")
|
||||
mode = proxlb_data["meta"]["balancing"].get("mode", "used")
|
||||
balanciness = proxlb_data["meta"]["balancing"].get("balanciness", 10)
|
||||
method_value = [node_meta[f"{method}_{mode}_percent"] for node_meta in proxlb_data["nodes"].values()]
|
||||
|
||||
if mode == "assigned":
|
||||
method_value = [node_meta[f"{method}_{mode}_percent"] for node_meta in proxlb_data["nodes"].values()]
|
||||
|
||||
if proxlb_data["meta"]["balancing"].get(f"{method}_threshold", None):
|
||||
threshold = proxlb_data["meta"]["balancing"].get(f"{method}_threshold")
|
||||
highest_usage_node = max(proxlb_data["nodes"].values(), key=lambda x: x[f"{method}_{mode}_percent"])
|
||||
highest_node_value = highest_usage_node[f"{method}_{mode}_percent"]
|
||||
|
||||
if highest_node_value >= threshold:
|
||||
logger.debug(f"Guest balancing is required. Highest {method} usage node {highest_usage_node['name']} is above the defined threshold of {threshold}% with a value of {highest_node_value}%.")
|
||||
proxlb_data["meta"]["balancing"]["balance"] = True
|
||||
else:
|
||||
logger.debug(f"Guest balancing is ok. Highest {method} usage node {highest_usage_node['name']} is below the defined threshold of {threshold}% with a value of {highest_node_value}%.")
|
||||
proxlb_data["meta"]["balancing"]["balance"] = False
|
||||
|
||||
else:
|
||||
logger.debug(f"No {method} threshold defined for balancing. Skipping threshold check.")
|
||||
|
||||
elif mode == "used":
|
||||
method_value = [node_meta[f"{method}_{mode}_percent"] for node_meta in proxlb_data["nodes"].values()]
|
||||
|
||||
if proxlb_data["meta"]["balancing"].get(f"{method}_threshold", None):
|
||||
threshold = proxlb_data["meta"]["balancing"].get(f"{method}_threshold")
|
||||
highest_usage_node = max(proxlb_data["nodes"].values(), key=lambda x: x[f"{method}_{mode}_percent"])
|
||||
highest_node_value = highest_usage_node[f"{method}_{mode}_percent"]
|
||||
|
||||
if highest_node_value >= threshold:
|
||||
logger.debug(f"Guest balancing is required. Highest {method} usage node {highest_usage_node['name']} is above the defined threshold of {threshold}% with a value of {highest_node_value}%.")
|
||||
proxlb_data["meta"]["balancing"]["balance"] = True
|
||||
else:
|
||||
logger.debug(f"Guest balancing is ok. Highest {method} usage node {highest_usage_node['name']} is below the defined threshold of {threshold}% with a value of {highest_node_value}%.")
|
||||
proxlb_data["meta"]["balancing"]["balance"] = False
|
||||
|
||||
else:
|
||||
logger.debug(f"No {method} threshold defined for balancing. Skipping threshold check.")
|
||||
|
||||
elif mode == "psi":
|
||||
method_value = [node_meta[f"{method}_pressure_full_spikes_percent"] for node_meta in proxlb_data["nodes"].values()]
|
||||
any_node_hot = any(node.get(f"{method}_pressure_hot", False) for node in proxlb_data["nodes"].values())
|
||||
any_guest_hot = any(node.get(f"{method}_pressure_hot", False) for node in proxlb_data["guests"].values())
|
||||
|
||||
if any_node_hot:
|
||||
logger.debug(f"Guest balancing is required. A node is marked as HOT based on {method} pressure metrics.")
|
||||
proxlb_data["meta"]["balancing"]["balance"] = True
|
||||
else:
|
||||
logger.debug(f"Guest balancing is ok. No node is marked as HOT based on {method} pressure metrics.")
|
||||
|
||||
if any_guest_hot:
|
||||
logger.debug(f"Guest balancing is required. A guest is marked as HOT based on {method} pressure metrics.")
|
||||
proxlb_data["meta"]["balancing"]["balance"] = True
|
||||
else:
|
||||
logger.debug(f"Guest balancing is ok. No guest is marked as HOT based on {method} pressure metrics.")
|
||||
|
||||
return proxlb_data
|
||||
|
||||
else:
|
||||
logger.critical(f"Unknown balancing mode: {mode} provided. Cannot get balanciness.")
|
||||
sys.exit(1)
|
||||
|
||||
method_value_highest = max(method_value)
|
||||
method_value_lowest = min(method_value)
|
||||
|
||||
@@ -159,7 +295,23 @@ class Calculations:
|
||||
# Filter by the defined methods and modes for balancing
|
||||
method = proxlb_data["meta"]["balancing"].get("method", "memory")
|
||||
mode = proxlb_data["meta"]["balancing"].get("mode", "used")
|
||||
lowest_usage_node = min(filtered_nodes, key=lambda x: x[f"{method}_{mode}_percent"])
|
||||
|
||||
if mode == "assigned":
|
||||
logger.debug(f"Get best node for balancing by assigned {method} resources.")
|
||||
lowest_usage_node = min(filtered_nodes, key=lambda x: x[f"{method}_{mode}_percent"])
|
||||
|
||||
elif mode == "used":
|
||||
logger.debug(f"Get best node for balancing by used {method} resources.")
|
||||
lowest_usage_node = min(filtered_nodes, key=lambda x: x[f"{method}_{mode}_percent"])
|
||||
|
||||
elif mode == "psi":
|
||||
logger.debug(f"Get best node for balancing by pressure of {method} resources.")
|
||||
lowest_usage_node = min(filtered_nodes, key=lambda x: x[f"{method}_pressure_full_spikes_percent"])
|
||||
|
||||
else:
|
||||
logger.critical(f"Unknown balancing mode: {mode} provided. Cannot get best node.")
|
||||
sys.exit(1)
|
||||
|
||||
proxlb_data["meta"]["balancing"]["balance_reason"] = 'resources'
|
||||
proxlb_data["meta"]["balancing"]["balance_next_node"] = lowest_usage_node["name"]
|
||||
|
||||
@@ -188,7 +340,7 @@ class Calculations:
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
logger.debug("Starting: get_most_free_node.")
|
||||
logger.debug("Starting: relocate_guests_on_maintenance_nodes.")
|
||||
proxlb_data["meta"]["balancing"]["balance_next_guest"] = ""
|
||||
|
||||
for guest_name in proxlb_data["groups"]["maintenance"]:
|
||||
@@ -199,7 +351,7 @@ class Calculations:
|
||||
Calculations.update_node_resources(proxlb_data)
|
||||
logger.warning(f"Warning: Balancing may not be perfect because guest {guest_name} was located on a node which is in maintenance mode.")
|
||||
|
||||
logger.debug("Finished: get_most_free_node.")
|
||||
logger.debug("Finished: relocate_guests_on_maintenance_nodes.")
|
||||
|
||||
@staticmethod
|
||||
def relocate_guests(proxlb_data: Dict[str, Any]):
|
||||
@@ -217,7 +369,12 @@ class Calculations:
|
||||
None
|
||||
"""
|
||||
logger.debug("Starting: relocate_guests.")
|
||||
if proxlb_data["meta"]["balancing"]["balance"] or proxlb_data["meta"]["balancing"].get("enforce_affinity", False):
|
||||
|
||||
# Balance only if it is required by:
|
||||
# - balanciness
|
||||
# - Affinity/Anti-Affinity rules
|
||||
# - Pinning rules
|
||||
if proxlb_data["meta"]["balancing"]["balance"] or proxlb_data["meta"]["balancing"].get("enforce_affinity", False) or proxlb_data["meta"]["balancing"].get("enforce_pinning", False):
|
||||
|
||||
if proxlb_data["meta"]["balancing"].get("balance", False):
|
||||
logger.debug("Balancing of guests will be performed. Reason: balanciness")
|
||||
@@ -225,15 +382,75 @@ class Calculations:
|
||||
if proxlb_data["meta"]["balancing"].get("enforce_affinity", False):
|
||||
logger.debug("Balancing of guests will be performed. Reason: enforce affinity balancing")
|
||||
|
||||
for group_name in proxlb_data["groups"]["affinity"]:
|
||||
if proxlb_data["meta"]["balancing"].get("enforce_pinning", False):
|
||||
logger.debug("Balancing of guests will be performed. Reason: enforce pinning balancing")
|
||||
|
||||
# We get initially the node with the most free resources and then
|
||||
# migrate all guests within the group to that node to ensure the
|
||||
# affinity.
|
||||
Calculations.get_most_free_node(proxlb_data)
|
||||
# Sort guests by used memory
|
||||
# Allows processing larger guests first or smaller guests first
|
||||
larger_first = proxlb_data.get("meta", {}).get("balancing", {}).get("balance_larger_guests_first", False)
|
||||
|
||||
if larger_first:
|
||||
logger.debug("Larger guests will be processed first. (Sorting descending by memory used)")
|
||||
else:
|
||||
logger.debug("Smaller guests will be processed first. (Sorting ascending by memory used)")
|
||||
|
||||
# Sort affinity groups by number of guests to avoid creating more migrations than needed
|
||||
# because of affinity-groups and use afterwards memory for defining smaller/larger guests
|
||||
sorted_guest_usage_groups = sorted(
|
||||
proxlb_data["groups"]["affinity"],
|
||||
key=lambda g: (
|
||||
proxlb_data["groups"]["affinity"][g]["counter"],
|
||||
-proxlb_data["groups"]["affinity"][g]["memory_used"]
|
||||
if larger_first
|
||||
else proxlb_data["groups"]["affinity"][g]["memory_used"],
|
||||
)
|
||||
)
|
||||
|
||||
# Iterate over all affinity groups
|
||||
for group_name in sorted_guest_usage_groups:
|
||||
|
||||
# Validate balanciness again before processing each group
|
||||
Calculations.get_balanciness(proxlb_data)
|
||||
logger.debug(proxlb_data["meta"]["balancing"]["balance"])
|
||||
|
||||
if (not proxlb_data["meta"]["balancing"]["balance"]) and (not proxlb_data["meta"]["balancing"].get("enforce_affinity", False)) and (not proxlb_data["meta"]["balancing"].get("enforce_pinning", False)):
|
||||
logger.debug("Skipping further guest relocations as balanciness is now ok.")
|
||||
break
|
||||
|
||||
for guest_name in proxlb_data["groups"]["affinity"][group_name]["guests"]:
|
||||
proxlb_data["meta"]["balancing"]["balance_next_guest"] = guest_name
|
||||
|
||||
# Stop moving guests if the source node is no longer the most loaded
|
||||
source_node = proxlb_data["guests"][guest_name]["node_current"]
|
||||
method = proxlb_data["meta"]["balancing"].get("method", "memory")
|
||||
mode = proxlb_data["meta"]["balancing"].get("mode", "used")
|
||||
highest_node = max(proxlb_data["nodes"].values(), key=lambda n: n[f"{method}_used_percent"])
|
||||
|
||||
if highest_node["name"] != source_node:
|
||||
logger.debug(f"Stopping relocation for guest {guest_name}: source node {source_node} is no longer the most loaded node.")
|
||||
break
|
||||
|
||||
if not Calculations.validate_node_resources(proxlb_data, guest_name):
|
||||
logger.warning(f"Skipping relocation of guest {guest_name} due to insufficient resources on target node {proxlb_data['meta']['balancing']['balance_next_node']}. This might affect affinity group {group_name}.")
|
||||
continue
|
||||
|
||||
if mode == 'psi':
|
||||
logger.debug(f"Evaluating guest relocation based on {mode} mode.")
|
||||
method = proxlb_data["meta"]["balancing"].get("method", "memory")
|
||||
processed_guests_psi = proxlb_data["meta"]["balancing"].setdefault("processed_guests_psi", [])
|
||||
unprocessed_guests_psi = [guest for guest in proxlb_data["guests"].values() if guest["name"] not in processed_guests_psi]
|
||||
|
||||
# Filter by the defined methods and modes for balancing
|
||||
highest_usage_guest = max(unprocessed_guests_psi, key=lambda x: x[f"{method}_pressure_full_spikes_percent"])
|
||||
|
||||
# Append guest to the psi based processed list of guests
|
||||
if highest_usage_guest["name"] == guest_name and guest_name not in proxlb_data["meta"]["balancing"]["processed_guests_psi"]:
|
||||
proxlb_data["meta"]["balancing"]["processed_guests_psi"].append(guest_name)
|
||||
proxlb_data["meta"]["balancing"]["balance_next_guest"] = guest_name
|
||||
|
||||
else:
|
||||
logger.debug(f"Evaluating guest relocation based on {mode} mode.")
|
||||
proxlb_data["meta"]["balancing"]["balance_next_guest"] = guest_name
|
||||
|
||||
Calculations.val_anti_affinity(proxlb_data, guest_name)
|
||||
Calculations.val_node_relationships(proxlb_data, guest_name)
|
||||
Calculations.update_node_resources(proxlb_data)
|
||||
@@ -266,23 +483,28 @@ class Calculations:
|
||||
if guest_name in proxlb_data["groups"]["anti_affinity"][group_name]['guests'] and not proxlb_data["guests"][guest_name]["processed"]:
|
||||
logger.debug(f"Anti-Affinity: Guest: {guest_name} is included in anti-affinity group: {group_name}.")
|
||||
|
||||
# Iterate over all available nodes
|
||||
for node_name in proxlb_data["nodes"].keys():
|
||||
# Check if the group has only one member. If so skip new guest node assignment.
|
||||
if proxlb_data["groups"]["anti_affinity"][group_name]["counter"] > 1:
|
||||
logger.debug(f"Anti-Affinity: Group has more than 1 member.")
|
||||
# Iterate over all available nodes
|
||||
for node_name in proxlb_data["nodes"].keys():
|
||||
|
||||
# Only select node if it was not used before and is not in a
|
||||
# maintenance mode. Afterwards, add it to the list of already
|
||||
# used nodes for the current anti-affinity group
|
||||
if node_name not in proxlb_data["groups"]["anti_affinity"][group_name]["used_nodes"]:
|
||||
# Only select node if it was not used before and is not in a
|
||||
# maintenance mode. Afterwards, add it to the list of already
|
||||
# used nodes for the current anti-affinity group
|
||||
if node_name not in proxlb_data["groups"]["anti_affinity"][group_name]["used_nodes"]:
|
||||
|
||||
if not proxlb_data["nodes"][node_name]["maintenance"]:
|
||||
# If the node has not been used yet, we assign this node to the guest
|
||||
proxlb_data["meta"]["balancing"]["balance_next_node"] = node_name
|
||||
proxlb_data["groups"]["anti_affinity"][group_name]["used_nodes"].append(node_name)
|
||||
logger.debug(f"Node: {node_name} marked as used for anti-affinity group: {group_name} with guest {guest_name}")
|
||||
break
|
||||
if not proxlb_data["nodes"][node_name]["maintenance"]:
|
||||
# If the node has not been used yet, we assign this node to the guest
|
||||
proxlb_data["meta"]["balancing"]["balance_next_node"] = node_name
|
||||
proxlb_data["groups"]["anti_affinity"][group_name]["used_nodes"].append(node_name)
|
||||
logger.debug(f"Node: {node_name} marked as used for anti-affinity group: {group_name} with guest {guest_name}")
|
||||
break
|
||||
|
||||
else:
|
||||
logger.critical(f"Node: {node_name} already got used for anti-affinity group:: {group_name}. (Tried for guest: {guest_name})")
|
||||
else:
|
||||
logger.critical(f"Node: {node_name} already got used for anti-affinity group:: {group_name}. (Tried for guest: {guest_name})")
|
||||
else:
|
||||
logger.debug(f"Anti-Affinity: Group has less than 2 members. Skipping node calculation for the group.")
|
||||
|
||||
else:
|
||||
logger.debug(f"Guest: {guest_name} is not included in anti-affinity group: {group_name}. Skipping.")
|
||||
@@ -307,8 +529,20 @@ class Calculations:
|
||||
if len(proxlb_data["guests"][guest_name]["node_relationships"]) > 0:
|
||||
logger.debug(f"Guest '{guest_name}' has relationships defined to node(s): {','.join(proxlb_data['guests'][guest_name]['node_relationships'])}. Pinning to node.")
|
||||
|
||||
# Get the node with the most free resources of the group
|
||||
# Get the list of nodes that are defined as relationship for the guest
|
||||
guest_node_relation_list = proxlb_data["guests"][guest_name]["node_relationships"]
|
||||
|
||||
# Validate if strict relationships are defined. If not, we prefer
|
||||
# the most free node in addition to the relationship list.
|
||||
if proxlb_data["guests"][guest_name]["node_relationships_strict"]:
|
||||
logger.debug(f"Guest '{guest_name}' has strict node relationships defined. Only nodes in the relationship list will be considered for pinning.")
|
||||
else:
|
||||
logger.debug(f"Guest '{guest_name}' has non-strict node relationships defined. Prefering nodes in the relationship list for pinning.")
|
||||
Calculations.get_most_free_node(proxlb_data)
|
||||
most_free_node = proxlb_data["meta"]["balancing"]["balance_next_node"]
|
||||
guest_node_relation_list.append(most_free_node)
|
||||
|
||||
# Get the most free node from the relationship list, or the most free node overall
|
||||
Calculations.get_most_free_node(proxlb_data, False, guest_node_relation_list)
|
||||
|
||||
# Validate if the specified node name is really part of the cluster
|
||||
@@ -323,7 +557,7 @@ class Calculations:
|
||||
logger.debug("Finished: val_node_relationships.")
|
||||
|
||||
@staticmethod
|
||||
def update_node_resources(proxlb_data):
|
||||
def update_node_resources(proxlb_data: Dict[str, Any]):
|
||||
"""
|
||||
Updates the resource allocation and usage statistics for nodes when a guest
|
||||
is moved from one node to another.
|
||||
@@ -343,6 +577,11 @@ class Calculations:
|
||||
"""
|
||||
logger.debug("Starting: update_node_resources.")
|
||||
guest_name = proxlb_data["meta"]["balancing"]["balance_next_guest"]
|
||||
|
||||
if guest_name == "":
|
||||
logger.debug("No guest defined to update node resources for.")
|
||||
return
|
||||
|
||||
node_current = proxlb_data["guests"][guest_name]["node_current"]
|
||||
node_target = proxlb_data["meta"]["balancing"]["balance_next_node"]
|
||||
|
||||
@@ -383,7 +622,217 @@ class Calculations:
|
||||
proxlb_data["nodes"][node_current]["disk_used_percent"] = proxlb_data["nodes"][node_current]["disk_used"] / proxlb_data["nodes"][node_current]["disk_total"] * 100
|
||||
|
||||
# Assign guest to the new target node
|
||||
proxlb_data["guests"][guest_name]["node_target"] = node_target
|
||||
logger.debug(f"Set guest {guest_name} from node {node_current} to node {node_target}.")
|
||||
if not proxlb_data["guests"][guest_name]["ignore"]:
|
||||
proxlb_data["guests"][guest_name]["node_target"] = node_target
|
||||
logger.debug(f"Set guest {guest_name} from node {node_current} to node {node_target}.")
|
||||
else:
|
||||
logger.debug(f"Guest {guest_name} is marked as ignored. Skipping target node assignment.")
|
||||
|
||||
Calculations.recalc_node_statistics(proxlb_data, node_target)
|
||||
Calculations.recalc_node_statistics(proxlb_data, node_current)
|
||||
|
||||
logger.debug("Finished: update_node_resources.")
|
||||
|
||||
def validate_affinity_map(proxlb_data: Dict[str, Any]):
|
||||
"""
|
||||
Validates the affinity and anti-affinity constraints for all guests in the ProxLB data structure.
|
||||
|
||||
This function iterates through each guest and checks both affinity and anti-affinity rules.
|
||||
If any guest violates these constraints, it sets the enforce_affinity flag to trigger rebalancing
|
||||
and skips further validation for efficiency.
|
||||
|
||||
Args:
|
||||
proxlb_data (Dict[str, Any]): A dictionary containing ProxLB configuration with the following structure:
|
||||
- "guests" (list): List of guest identifiers to validate
|
||||
- "meta" (dict): Metadata dictionary containing:
|
||||
- "balancing" (dict): Balancing configuration with "enforce_affinity" flag
|
||||
|
||||
Returns:
|
||||
None: Modifies proxlb_data in-place by updating the "enforce_affinity" flag in meta.balancing
|
||||
|
||||
Raises:
|
||||
None: Function handles validation gracefully and logs outcomes
|
||||
"""
|
||||
logger.debug("Starting: validate_current_affinity.")
|
||||
balancing_ok = True
|
||||
|
||||
for guest in proxlb_data["guests"]:
|
||||
|
||||
# We do not need to validate anymore if rebalancing is required
|
||||
if balancing_ok is False:
|
||||
proxlb_data["meta"]["balancing"]["enforce_affinity"] = True
|
||||
logger.debug(f"Rebalancing based on affinity/anti-affinity map is required. Skipping further validation...")
|
||||
break
|
||||
|
||||
balancing_state_affinity = Calculations.validate_current_affinity(proxlb_data, guest)
|
||||
balancing_state_anti_affinity = Calculations.validate_current_anti_affinity(proxlb_data, guest)
|
||||
logger.debug(f"Affinity for guest {guest} is {'valid' if balancing_state_affinity else 'NOT valid'}")
|
||||
logger.debug(f"Anti-affinity for guest {guest} is {'valid' if balancing_state_anti_affinity else 'NOT valid'}")
|
||||
|
||||
balancing_ok = balancing_state_affinity and balancing_state_anti_affinity
|
||||
|
||||
if balancing_ok:
|
||||
logger.debug(f"Rebalancing based on affinity/anti-affinity map is not required.")
|
||||
proxlb_data["meta"]["balancing"]["enforce_affinity"] = False
|
||||
|
||||
logger.debug("Finished: validate_current_affinity.")
|
||||
|
||||
@staticmethod
|
||||
def get_guest_node(proxlb_data: Dict[str, Any], guest_name: str) -> str:
|
||||
"""
|
||||
Return a currently assoicated PVE node where the guest is running on.
|
||||
|
||||
Args:
|
||||
proxlb_data (Dict[str, Any]): A dictionary containing ProxLB configuration.
|
||||
|
||||
Returns:
|
||||
node_name_current (str): The name of the current node where the guest runs on.
|
||||
|
||||
"""
|
||||
return proxlb_data["guests"][guest_name]["node_current"]
|
||||
|
||||
@staticmethod
|
||||
def validate_current_affinity(proxlb_data: Dict[str, Any], guest_name: str) -> bool:
|
||||
"""
|
||||
Validate that all guests in affinity groups containing the specified guest are on the same non-maintenance node.
|
||||
|
||||
This function checks affinity group constraints for a given guest. It ensures that:
|
||||
1. All guests within an affinity group are located on the same physical node
|
||||
2. The node hosting the affinity group is not in maintenance mode
|
||||
|
||||
Args:
|
||||
proxlb_data (Dict[str, Any]): A dictionary containing the complete ProxLB state including:
|
||||
- "groups": Dictionary with "affinity" key containing affinity group definitions
|
||||
- "guests": Dictionary with guest information
|
||||
- "nodes": Dictionary with node information including maintenance status
|
||||
guest_name (str): The name of the guest to validate affinity for
|
||||
|
||||
Returns:
|
||||
bool: True if all affinity groups containing the guest are valid (all members on same
|
||||
non-maintenance node), False otherwise
|
||||
"""
|
||||
logger.debug("Starting: validate_current_affinity.")
|
||||
for group_name, grp in proxlb_data["groups"]["affinity"].items():
|
||||
if guest_name not in grp["guests"]:
|
||||
continue
|
||||
|
||||
nodes = []
|
||||
for group in grp["guests"]:
|
||||
if group not in proxlb_data["guests"]:
|
||||
continue
|
||||
|
||||
node = Calculations.get_guest_node(proxlb_data, group)
|
||||
if proxlb_data["nodes"][node]["maintenance"]:
|
||||
logger.debug(f"Group '{group_name}' invalid: node '{node}' in maintenance.")
|
||||
return False
|
||||
nodes.append(node)
|
||||
|
||||
if len(set(nodes)) != 1:
|
||||
logger.debug(f"Group '{group_name}' invalid: guests spread across nodes {set(nodes)}.")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def validate_current_anti_affinity(proxlb_data: Dict[str, Any], guest_name: str) -> bool:
|
||||
"""
|
||||
Validate that all guests in anti-affinity groups containing the specified guest are not on the same node.
|
||||
|
||||
This function checks anti-affinity group constraints for a given guest. It ensures that:
|
||||
1. All guests within an anti-affinity group are located on the same physical node
|
||||
2. The node hosting the anti-affinity group is not in maintenance mode
|
||||
|
||||
Args:
|
||||
proxlb_data (Dict[str, Any]): A dictionary containing the complete ProxLB state including:
|
||||
- "groups": Dictionary with "affinity" key containing affinity group definitions
|
||||
- "guests": Dictionary with guest information
|
||||
- "nodes": Dictionary with node information including maintenance status
|
||||
guest_name (str): The name of the guest to validate affinity for
|
||||
|
||||
Returns:
|
||||
bool: True if all anti-affinity groups containing the guest are valid (all members on different
|
||||
non-maintenance node), False otherwise
|
||||
"""
|
||||
logger.debug("Starting: validate_current_anti_affinity.")
|
||||
for group_name, grp in proxlb_data["groups"]["anti_affinity"].items():
|
||||
if guest_name not in grp["guests"]:
|
||||
continue
|
||||
nodes = []
|
||||
for group in grp["guests"]:
|
||||
if group not in proxlb_data["guests"]:
|
||||
continue
|
||||
|
||||
node = Calculations.get_guest_node(proxlb_data, group)
|
||||
if proxlb_data["nodes"][node]["maintenance"]:
|
||||
return False
|
||||
nodes.append(node)
|
||||
|
||||
if len(nodes) != len(set(nodes)):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def validate_node_resources(proxlb_data: Dict[str, Any], guest_name: str) -> bool:
|
||||
"""
|
||||
Validate that the target node has sufficient resources to host the specified guest.
|
||||
|
||||
This function checks if the target node, determined by the balancing logic,
|
||||
has enough CPU, memory, and disk resources available to accommodate the guest.
|
||||
|
||||
Args:
|
||||
proxlb_data (Dict[str, Any]): A dictionary containing the complete ProxLB state including:
|
||||
- "nodes": Dictionary with node resource information
|
||||
- "guests": Dictionary with guest resource requirements
|
||||
- "meta": Dictionary with balancing information including target node
|
||||
guest_name (str): The name of the guest to validate resources for
|
||||
Returns:
|
||||
bool: True if the target node has sufficient resources, False otherwise
|
||||
"""
|
||||
logger.debug("Starting: validate_node_resources.")
|
||||
node_target = proxlb_data["meta"]["balancing"]["balance_next_node"]
|
||||
|
||||
node_memory_free = proxlb_data["nodes"][node_target]["memory_free"]
|
||||
node_cpu_free = proxlb_data["nodes"][node_target]["cpu_free"]
|
||||
node_disk_free = proxlb_data["nodes"][node_target]["disk_free"]
|
||||
|
||||
guest_memory_required = proxlb_data["guests"][guest_name]["memory_used"]
|
||||
guest_cpu_required = proxlb_data["guests"][guest_name]["cpu_used"]
|
||||
guest_disk_required = proxlb_data["guests"][guest_name]["disk_used"]
|
||||
|
||||
if guest_memory_required < node_memory_free:
|
||||
logger.debug(f"Node '{node_target}' has sufficient resources ({node_memory_free / (1024 ** 3):.2f} GB free) for guest '{guest_name}'.")
|
||||
logger.debug("Finished: validate_node_resources.")
|
||||
return True
|
||||
else:
|
||||
logger.debug(f"Node '{node_target}' lacks sufficient resources ({node_memory_free / (1024 ** 3):.2f} GB free) for guest '{guest_name}'.")
|
||||
logger.debug("Finished: validate_node_resources.")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def recalc_node_statistics(proxlb_data: Dict[str, Any], node_name: str) -> None:
|
||||
"""
|
||||
Recalculates node statistics including free resources and usage percentages.
|
||||
|
||||
This function updates the computed statistics for a node based on its current
|
||||
resource allocation and usage. It calculates free resources, usage percentages,
|
||||
and assigned percentages for CPU, memory, and disk.
|
||||
|
||||
Args:
|
||||
proxlb_data (Dict[str, Any]): A dictionary containing the complete ProxLB state including:
|
||||
- "nodes": Dictionary with node resource information
|
||||
node_name (str): The name of the node to recalculate statistics for
|
||||
|
||||
Returns:
|
||||
None: Modifies proxlb_data in-place by updating node statistics
|
||||
"""
|
||||
n = proxlb_data["nodes"][node_name]
|
||||
n["cpu_free"] = max(0, n["cpu_total"] - n["cpu_used"])
|
||||
n["memory_free"] = max(0, n["memory_total"] - n["memory_used"])
|
||||
n["disk_free"] = max(0, n["disk_total"] - n["disk_used"])
|
||||
n["cpu_used_percent"] = (n["cpu_used"] / n["cpu_total"] * 100) if n["cpu_total"] else 0
|
||||
n["memory_used_percent"] = (n["memory_used"] / n["memory_total"] * 100) if n["memory_total"] else 0
|
||||
n["disk_used_percent"] = (n["disk_used"] / n["disk_total"] * 100) if n["disk_total"] else 0
|
||||
n["cpu_assigned_percent"] = (n["cpu_assigned"] / n["cpu_total"] * 100) if n["cpu_total"] else 0
|
||||
n["memory_assigned_percent"] = (n["memory_assigned"] / n["memory_total"] * 100) if n["memory_total"] else 0
|
||||
n["disk_assigned_percent"] = (n["disk_assigned"] / n["disk_total"] * 100) if n["disk_total"] else 0
|
||||
|
||||
124
proxlb/models/features.py
Normal file
124
proxlb/models/features.py
Normal file
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
ProxLB Features module for validating and adjusting feature flags
|
||||
based on Proxmox VE node versions and cluster compatibility.
|
||||
"""
|
||||
|
||||
__author__ = "Florian Paul Azim Hoberg <gyptazy>"
|
||||
__copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
|
||||
__license__ = "GPL-3.0"
|
||||
|
||||
|
||||
from typing import List
|
||||
from typing import Dict, Any
|
||||
from utils.logger import SystemdLogger
|
||||
from packaging import version
|
||||
|
||||
logger = SystemdLogger()
|
||||
|
||||
|
||||
class Features:
|
||||
"""
|
||||
ProxLB Features module for validating and adjusting feature flags
|
||||
based on Proxmox VE node versions and cluster compatibility.
|
||||
|
||||
Responsibilities:
|
||||
- Validate and adjust feature flags based on Proxmox VE node versions.
|
||||
|
||||
Methods:
|
||||
__init__():
|
||||
No-op initializer.
|
||||
|
||||
validate_available_features(proxlb_data: dict) -> None:
|
||||
Static method that inspects proxlb_data["nodes"] versions and disables
|
||||
incompatible balancing features for Proxmox VE versions < 9.0.0.
|
||||
This function mutates proxlb_data in place.
|
||||
|
||||
Notes:
|
||||
- Expects proxlb_data to be a dict with "nodes" and "meta" keys.
|
||||
"""
|
||||
def __init__(self):
|
||||
"""
|
||||
Initializes the Features class.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def validate_available_features(proxlb_data: any) -> None:
|
||||
"""
|
||||
Validate and adjust feature flags in the provided proxlb_data according to Proxmox VE versions.
|
||||
|
||||
This function inspects the cluster node versions in proxlb_data and disables features
|
||||
that are incompatible with Proxmox VE versions older than 9.0.0. Concretely, if any node
|
||||
reports a 'pve_version' lower than "9.0.0":
|
||||
- If meta.balancing.with_conntrack_state is truthy, it is set to False and a warning is logged.
|
||||
- If meta.balancing.mode equals "psi", meta.balancing.enable is set to False and a warning is logged.
|
||||
|
||||
proxlb_data (dict): Cluster data structure that must contain:
|
||||
- "nodes": a mapping (e.g., dict) whose values are mappings containing a 'pve_version' string.
|
||||
- "meta": a mapping that may contain a "balancing" mapping with keys:
|
||||
- "with_conntrack_state" (bool, optional)
|
||||
- "mode" (str, optional)
|
||||
- "enable" (bool, optional)
|
||||
|
||||
None: The function mutates proxlb_data in place to disable incompatible features.
|
||||
|
||||
Side effects:
|
||||
- Mutates proxlb_data["meta"]["balancing"] when incompatible features are detected.
|
||||
- Emits debug and warning log messages.
|
||||
|
||||
Notes:
|
||||
- Unexpected or missing keys/types in proxlb_data may raise KeyError or TypeError.
|
||||
- Version comparison uses semantic version parsing; callers should provide versions as strings.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
logger.debug("Starting: validate_available_features.")
|
||||
|
||||
any_non_pve9_node = any(version.parse(n['pve_version']) < version.parse("9.0.0") for n in proxlb_data["nodes"].values())
|
||||
if any_non_pve9_node:
|
||||
|
||||
with_conntrack_state = proxlb_data["meta"].get("balancing", {}).get("with_conntrack_state", False)
|
||||
if with_conntrack_state:
|
||||
logger.warning("Non Proxmox VE 9 systems detected: Deactivating migration option 'with-conntrack-state'!")
|
||||
proxlb_data["meta"]["balancing"]["with_conntrack_state"] = False
|
||||
|
||||
psi_balancing = proxlb_data["meta"].get("balancing", {}).get("mode", None)
|
||||
if psi_balancing == "psi":
|
||||
logger.warning("Non Proxmox VE 9 systems detected: Deactivating balancing!")
|
||||
proxlb_data["meta"]["balancing"]["enable"] = False
|
||||
|
||||
logger.debug("Finished: validate_available_features.")
|
||||
|
||||
@staticmethod
|
||||
def validate_any_non_pve9_node(meta: any, nodes: any) -> dict:
|
||||
"""
|
||||
Validate if any node in the cluster is running Proxmox VE < 9.0.0 and update meta accordingly.
|
||||
|
||||
This function inspects the cluster node versions and sets a flag in meta indicating whether
|
||||
any node is running a Proxmox VE version older than 9.0.0.
|
||||
|
||||
Args:
|
||||
meta (dict): Metadata structure that will be updated with cluster version information.
|
||||
nodes (dict): Cluster nodes mapping whose values contain 'pve_version' strings.
|
||||
|
||||
Returns:
|
||||
dict: The updated meta dictionary with 'cluster_non_pve9' flag set to True or False.
|
||||
|
||||
Side effects:
|
||||
- Mutates meta["meta"]["cluster_non_pve9"] based on node versions.
|
||||
- Emits debug log messages.
|
||||
|
||||
Notes:
|
||||
- Version comparison uses semantic version parsing; defaults to "0.0.0" if pve_version is missing.
|
||||
"""
|
||||
logger.debug("Starting: validate_any_non_pve9_node.")
|
||||
any_non_pve9_node = any(version.parse(node.get("pve_version", "0.0.0")) < version.parse("9.0.0") for node in nodes.get("nodes", {}).values())
|
||||
|
||||
if any_non_pve9_node:
|
||||
meta["meta"]["cluster_non_pve9"] = True
|
||||
logger.debug("Finished: validate_any_non_pve9_node. Result: True")
|
||||
else:
|
||||
meta["meta"]["cluster_non_pve9"] = False
|
||||
logger.debug("Finished: validate_any_non_pve9_node. Result: False")
|
||||
|
||||
return meta
|
||||
@@ -10,6 +10,8 @@ __license__ = "GPL-3.0"
|
||||
|
||||
from typing import Dict, Any
|
||||
from utils.logger import SystemdLogger
|
||||
from models.pools import Pools
|
||||
from models.ha_rules import HaRules
|
||||
from models.tags import Tags
|
||||
import time
|
||||
|
||||
@@ -35,7 +37,7 @@ class Guests:
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_guests(proxmox_api: any, nodes: Dict[str, Any], meta: Dict[str, Any]) -> Dict[str, Any]:
|
||||
def get_guests(proxmox_api: any, pools: Dict[str, Any], ha_rules: Dict[str, Any], nodes: Dict[str, Any], meta: Dict[str, Any], proxlb_config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Get metrics of all guests in a Proxmox cluster.
|
||||
|
||||
@@ -45,7 +47,11 @@ class Guests:
|
||||
|
||||
Args:
|
||||
proxmox_api (any): The Proxmox API client instance.
|
||||
pools (Dict[str, Any]): A dictionary containing information about the pools in the Proxmox cluster.
|
||||
ha_rules (Dict[str, Any]): A dictionary containing information about the HA rules in the
|
||||
nodes (Dict[str, Any]): A dictionary containing information about the nodes in the Proxmox cluster.
|
||||
meta (Dict[str, Any]): A dictionary containing metadata information.
|
||||
proxmox_config (Dict[str, Any]): A dictionary containing the ProxLB configuration.
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: A dictionary containing metrics and information for all running guests.
|
||||
@@ -62,24 +68,42 @@ class Guests:
|
||||
# resource metrics for rebalancing to ensure that we do not overprovisiong the node.
|
||||
for guest in proxmox_api.nodes(node).qemu.get():
|
||||
if guest['status'] == 'running':
|
||||
|
||||
guests['guests'][guest['name']] = {}
|
||||
guests['guests'][guest['name']]['name'] = guest['name']
|
||||
guests['guests'][guest['name']]['cpu_total'] = int(guest['cpus'])
|
||||
guests['guests'][guest['name']]['cpu_used'] = Guests.get_guest_cpu_usage(proxmox_api, node, guest['vmid'], guest['name'])
|
||||
guests['guests'][guest['name']]['cpu_used'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', None)
|
||||
guests['guests'][guest['name']]['cpu_pressure_some_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'some')
|
||||
guests['guests'][guest['name']]['cpu_pressure_full_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'full')
|
||||
guests['guests'][guest['name']]['cpu_pressure_some_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'some', spikes=True)
|
||||
guests['guests'][guest['name']]['cpu_pressure_full_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'full', spikes=True)
|
||||
guests['guests'][guest['name']]['cpu_pressure_hot'] = False
|
||||
guests['guests'][guest['name']]['memory_total'] = guest['maxmem']
|
||||
guests['guests'][guest['name']]['memory_used'] = guest['mem']
|
||||
guests['guests'][guest['name']]['memory_pressure_some_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'some')
|
||||
guests['guests'][guest['name']]['memory_pressure_full_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'full')
|
||||
guests['guests'][guest['name']]['memory_pressure_some_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'some', spikes=True)
|
||||
guests['guests'][guest['name']]['memory_pressure_full_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'full', spikes=True)
|
||||
guests['guests'][guest['name']]['memory_pressure_hot'] = False
|
||||
guests['guests'][guest['name']]['disk_total'] = guest['maxdisk']
|
||||
guests['guests'][guest['name']]['disk_used'] = guest['disk']
|
||||
guests['guests'][guest['name']]['disk_pressure_some_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'some')
|
||||
guests['guests'][guest['name']]['disk_pressure_full_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'full')
|
||||
guests['guests'][guest['name']]['disk_pressure_some_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'some', spikes=True)
|
||||
guests['guests'][guest['name']]['disk_pressure_full_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'full', spikes=True)
|
||||
guests['guests'][guest['name']]['disk_pressure_hot'] = False
|
||||
guests['guests'][guest['name']]['id'] = guest['vmid']
|
||||
guests['guests'][guest['name']]['node_current'] = node
|
||||
guests['guests'][guest['name']]['node_target'] = node
|
||||
guests['guests'][guest['name']]['processed'] = False
|
||||
guests['guests'][guest['name']]['pressure_hot'] = False
|
||||
guests['guests'][guest['name']]['tags'] = Tags.get_tags_from_guests(proxmox_api, node, guest['vmid'], 'vm')
|
||||
guests['guests'][guest['name']]['affinity_groups'] = Tags.get_affinity_groups(guests['guests'][guest['name']]['tags'])
|
||||
guests['guests'][guest['name']]['anti_affinity_groups'] = Tags.get_anti_affinity_groups(guests['guests'][guest['name']]['tags'])
|
||||
guests['guests'][guest['name']]['pools'] = Pools.get_pools_for_guest(guest['name'], pools)
|
||||
guests['guests'][guest['name']]['ha_rules'] = HaRules.get_ha_rules_for_guest(guest['name'], ha_rules, guest['vmid'])
|
||||
guests['guests'][guest['name']]['affinity_groups'] = Tags.get_affinity_groups(guests['guests'][guest['name']]['tags'], guests['guests'][guest['name']]['pools'], guests['guests'][guest['name']]['ha_rules'], proxlb_config)
|
||||
guests['guests'][guest['name']]['anti_affinity_groups'] = Tags.get_anti_affinity_groups(guests['guests'][guest['name']]['tags'], guests['guests'][guest['name']]['pools'], guests['guests'][guest['name']]['ha_rules'], proxlb_config)
|
||||
guests['guests'][guest['name']]['ignore'] = Tags.get_ignore(guests['guests'][guest['name']]['tags'])
|
||||
guests['guests'][guest['name']]['node_relationships'] = Tags.get_node_relationships(guests['guests'][guest['name']]['tags'])
|
||||
guests['guests'][guest['name']]['node_relationships'] = Tags.get_node_relationships(guests['guests'][guest['name']]['tags'], nodes, guests['guests'][guest['name']]['pools'], guests['guests'][guest['name']]['ha_rules'], proxlb_config)
|
||||
guests['guests'][guest['name']]['node_relationships_strict'] = Pools.get_pool_node_affinity_strictness(proxlb_config, guests['guests'][guest['name']]['pools'])
|
||||
guests['guests'][guest['name']]['type'] = 'vm'
|
||||
|
||||
logger.debug(f"Resources of Guest {guest['name']} (type VM) added: {guests['guests'][guest['name']]}")
|
||||
@@ -94,20 +118,39 @@ class Guests:
|
||||
guests['guests'][guest['name']] = {}
|
||||
guests['guests'][guest['name']]['name'] = guest['name']
|
||||
guests['guests'][guest['name']]['cpu_total'] = int(guest['cpus'])
|
||||
guests['guests'][guest['name']]['cpu_used'] = Guests.get_guest_cpu_usage(proxmox_api, node, guest['vmid'], guest['name'])
|
||||
guests['guests'][guest['name']]['cpu_used'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', None)
|
||||
guests['guests'][guest['name']]['cpu_pressure_some_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'some')
|
||||
guests['guests'][guest['name']]['cpu_pressure_full_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'full')
|
||||
guests['guests'][guest['name']]['cpu_pressure_some_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'some', spikes=True)
|
||||
guests['guests'][guest['name']]['cpu_pressure_full_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'full', spikes=True)
|
||||
guests['guests'][guest['name']]['cpu_pressure_hot'] = False
|
||||
guests['guests'][guest['name']]['memory_total'] = guest['maxmem']
|
||||
guests['guests'][guest['name']]['memory_used'] = guest['mem']
|
||||
guests['guests'][guest['name']]['memory_pressure_some_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'some')
|
||||
guests['guests'][guest['name']]['memory_pressure_full_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'full')
|
||||
guests['guests'][guest['name']]['memory_pressure_some_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'some', spikes=True)
|
||||
guests['guests'][guest['name']]['memory_pressure_full_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'full', spikes=True)
|
||||
guests['guests'][guest['name']]['memory_pressure_hot'] = False
|
||||
guests['guests'][guest['name']]['disk_total'] = guest['maxdisk']
|
||||
guests['guests'][guest['name']]['disk_used'] = guest['disk']
|
||||
guests['guests'][guest['name']]['disk_pressure_some_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'some')
|
||||
guests['guests'][guest['name']]['disk_pressure_full_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'full')
|
||||
guests['guests'][guest['name']]['disk_pressure_some_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'some', spikes=True)
|
||||
guests['guests'][guest['name']]['disk_pressure_full_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'full', spikes=True)
|
||||
guests['guests'][guest['name']]['disk_pressure_hot'] = False
|
||||
guests['guests'][guest['name']]['id'] = guest['vmid']
|
||||
guests['guests'][guest['name']]['node_current'] = node
|
||||
guests['guests'][guest['name']]['node_target'] = node
|
||||
guests['guests'][guest['name']]['processed'] = False
|
||||
guests['guests'][guest['name']]['pressure_hot'] = False
|
||||
guests['guests'][guest['name']]['tags'] = Tags.get_tags_from_guests(proxmox_api, node, guest['vmid'], 'ct')
|
||||
guests['guests'][guest['name']]['affinity_groups'] = Tags.get_affinity_groups(guests['guests'][guest['name']]['tags'])
|
||||
guests['guests'][guest['name']]['anti_affinity_groups'] = Tags.get_anti_affinity_groups(guests['guests'][guest['name']]['tags'])
|
||||
guests['guests'][guest['name']]['pools'] = Pools.get_pools_for_guest(guest['name'], pools)
|
||||
guests['guests'][guest['name']]['ha_rules'] = HaRules.get_ha_rules_for_guest(guest['name'], ha_rules, guest['vmid'])
|
||||
guests['guests'][guest['name']]['affinity_groups'] = Tags.get_affinity_groups(guests['guests'][guest['name']]['tags'], guests['guests'][guest['name']]['pools'], guests['guests'][guest['name']]['ha_rules'], proxlb_config)
|
||||
guests['guests'][guest['name']]['anti_affinity_groups'] = Tags.get_anti_affinity_groups(guests['guests'][guest['name']]['tags'], guests['guests'][guest['name']]['pools'], guests['guests'][guest['name']]['ha_rules'], proxlb_config)
|
||||
guests['guests'][guest['name']]['ignore'] = Tags.get_ignore(guests['guests'][guest['name']]['tags'])
|
||||
guests['guests'][guest['name']]['node_relationships'] = Tags.get_node_relationships(guests['guests'][guest['name']]['tags'])
|
||||
guests['guests'][guest['name']]['node_relationships'] = Tags.get_node_relationships(guests['guests'][guest['name']]['tags'], nodes, guests['guests'][guest['name']]['pools'], guests['guests'][guest['name']]['ha_rules'], proxlb_config)
|
||||
guests['guests'][guest['name']]['node_relationships_strict'] = Pools.get_pool_node_affinity_strictness(proxlb_config, guests['guests'][guest['name']]['pools'])
|
||||
guests['guests'][guest['name']]['type'] = 'ct'
|
||||
|
||||
logger.debug(f"Resources of Guest {guest['name']} (type CT) added: {guests['guests'][guest['name']]}")
|
||||
@@ -118,36 +161,55 @@ class Guests:
|
||||
return guests
|
||||
|
||||
@staticmethod
|
||||
def get_guest_cpu_usage(proxmox_api, node_name: str, vm_id: int, vm_name: str) -> float:
|
||||
def get_guest_rrd_data(proxmox_api, node_name: str, vm_id: int, vm_name: str, object_name: str, object_type: str, spikes=False) -> float:
|
||||
"""
|
||||
Retrieve the average CPU usage of a guest instance (VM/CT) over the past hour.
|
||||
|
||||
This method queries the Proxmox VE API for RRD (Round-Robin Database) data
|
||||
related to CPU usage of a specific guest instance and calculates the average CPU usage
|
||||
over the last hour using the "AVERAGE" consolidation function.
|
||||
Retrieves the rrd data metrics for a specific resource (CPU, memory, disk) of a guest VM or CT.
|
||||
|
||||
Args:
|
||||
proxmox_api: An instance of the Proxmox API client.
|
||||
node_name (str): The name of the Proxmox node hosting the VM.
|
||||
vm_id (int): The unique identifier of the guest instance (VM/CT).
|
||||
vm_name (str): The name of the guest instance (VM/CT).
|
||||
proxmox_api (Any): The Proxmox API client instance.
|
||||
node_name (str): The name of the node hosting the guest.
|
||||
vm_id (int): The ID of the guest VM or CT.
|
||||
vm_name (str): The name of the guest VM or CT.
|
||||
object_name (str): The resource type to query (e.g., 'cpu', 'memory', 'disk').
|
||||
object_type (str, optional): The pressure type ('some', 'full') or None for average usage.
|
||||
spikes (bool, optional): Whether to consider spikes in the calculation. Defaults to False.
|
||||
|
||||
Returns:
|
||||
float: The average CPU usage as a fraction (0.0 to 1.0) over the past hour.
|
||||
Returns 0.0 if no data is available.
|
||||
float: The calculated average usage value for the specified resource.
|
||||
"""
|
||||
logger.debug("Finished: get_guest_cpu_usage.")
|
||||
logger.debug("Starting: get_guest_rrd_data.")
|
||||
time.sleep(0.1)
|
||||
|
||||
try:
|
||||
logger.debug(f"Getting RRD dara for guest: {vm_name}.")
|
||||
guest_data_rrd = proxmox_api.nodes(node_name).qemu(vm_id).rrddata.get(timeframe="hour", cf="AVERAGE")
|
||||
if spikes:
|
||||
logger.debug(f"Getting spike RRD data for {object_name} from guest: {vm_name}.")
|
||||
guest_data_rrd = proxmox_api.nodes(node_name).qemu(vm_id).rrddata.get(timeframe="hour", cf="MAX")
|
||||
else:
|
||||
logger.debug(f"Getting average RRD data for {object_name} from guest: {vm_name}.")
|
||||
guest_data_rrd = proxmox_api.nodes(node_name).qemu(vm_id).rrddata.get(timeframe="hour", cf="AVERAGE")
|
||||
except Exception:
|
||||
logger.error(f"Failed to retrieve RRD data for guest: {vm_name} (ID: {vm_id}) on node: {node_name}. Using 0.0 as CPU usage.")
|
||||
logger.debug("Finished: get_guest_cpu_usage.")
|
||||
return 0.0
|
||||
logger.error(f"Failed to retrieve RRD data for guest: {vm_name} (ID: {vm_id}) on node: {node_name}. Using 0.0 as value.")
|
||||
logger.debug("Finished: get_guest_rrd_data.")
|
||||
return float(0.0)
|
||||
|
||||
cpu_usage = sum(entry.get("cpu", 0.0) for entry in guest_data_rrd) / len(guest_data_rrd)
|
||||
logger.debug(f"CPU RRD data for guest: {vm_name}: {cpu_usage}")
|
||||
logger.debug("Finished: get_guest_cpu_usage.")
|
||||
return cpu_usage
|
||||
if object_type:
|
||||
|
||||
lookup_key = f"pressure{object_name}{object_type}"
|
||||
if spikes:
|
||||
# RRD data is collected every minute, so we look at the last 6 entries
|
||||
# and take the maximum value to represent the spike
|
||||
logger.debug(f"Getting RRD data (spike: {spikes}) of pressure for {object_name} {object_type} from guest: {vm_name}.")
|
||||
rrd_data_value = [row.get(lookup_key) for row in guest_data_rrd if row.get(lookup_key) is not None]
|
||||
rrd_data_value = max(rrd_data_value[-6:], default=0.0)
|
||||
else:
|
||||
# Calculate the average value from the RRD data entries
|
||||
logger.debug(f"Getting RRD data (spike: {spikes}) of pressure for {object_name} {object_type} from guest: {vm_name}.")
|
||||
rrd_data_value = sum(entry.get(lookup_key, 0.0) for entry in guest_data_rrd) / len(guest_data_rrd)
|
||||
|
||||
else:
|
||||
logger.debug(f"Getting RRD data of cpu usage from guest: {vm_name}.")
|
||||
rrd_data_value = sum(entry.get("cpu", 0.0) for entry in guest_data_rrd) / len(guest_data_rrd)
|
||||
|
||||
logger.debug(f"RRD data (spike: {spikes}) for {object_name} from guest: {vm_name}: {rrd_data_value}")
|
||||
logger.debug("Finished: get_guest_rrd_data.")
|
||||
return rrd_data_value
|
||||
|
||||
126
proxlb/models/ha_rules.py
Normal file
126
proxlb/models/ha_rules.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
The HaRules class retrieves all HA rules defined on a Proxmox cluster
|
||||
including their affinity settings and member resources.
|
||||
"""
|
||||
|
||||
__author__ = "Florian Paul Azim Hoberg <gyptazy>"
|
||||
__copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
|
||||
__license__ = "GPL-3.0"
|
||||
|
||||
|
||||
from typing import Dict, Any
|
||||
from utils.logger import SystemdLogger
|
||||
|
||||
logger = SystemdLogger()
|
||||
|
||||
|
||||
class HaRules:
|
||||
"""
|
||||
The HaRules class retrieves all HA rules defined on a Proxmox cluster
|
||||
including their (anti)a-ffinity settings and member resources and translates
|
||||
them into a ProxLB usable format.
|
||||
|
||||
Methods:
|
||||
__init__:
|
||||
Initializes the HaRules class.
|
||||
|
||||
get_ha_rules(proxmox_api: any) -> Dict[str, Any]:
|
||||
Retrieve HA rule definitions from the Proxmox cluster.
|
||||
Returns a dict with a top-level "ha_rules" mapping each rule id to
|
||||
{"rule": <rule_id>, "type": <affinity_type>, "members": [<resource_ids>...]}.
|
||||
Converts affinity settings to descriptive format (affinity or anti-affinity).
|
||||
"""
|
||||
def __init__(self):
|
||||
"""
|
||||
Initializes the HA Rules class with the provided ProxLB data.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_ha_rules(proxmox_api: any, meta: dict) -> Dict[str, Any]:
|
||||
"""
|
||||
Retrieve all HA rules from a Proxmox cluster.
|
||||
|
||||
Queries the Proxmox API for HA rule definitions and returns a dictionary
|
||||
containing each rule's id, affinity type, and member resources (VM/CT IDs).
|
||||
This function processes rule affinity settings and converts them to a more
|
||||
descriptive format (affinity or anti-affinity).
|
||||
|
||||
Args:
|
||||
proxmox_api (any): Proxmox API client instance.
|
||||
meta (dict): The metadata dictionary containing cluster information.
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: Dictionary with a top-level "ha_rules" key mapping rule id
|
||||
to {"rule": <rule_id>, "type": <affinity_type>, "members": [<resource_ids>...]}.
|
||||
"""
|
||||
logger.debug("Starting: get_ha_rules.")
|
||||
ha_rules = {"ha_rules": {}}
|
||||
|
||||
# If any node is non PVE 9, skip fetching HA rules as they are unsupported
|
||||
if meta["meta"]["cluster_non_pve9"]:
|
||||
logger.debug("Skipping HA rule retrieval as non Proxmox VE 9 systems detected.")
|
||||
return ha_rules
|
||||
else:
|
||||
logger.debug("Cluster running Proxmox VE 9 or newer, proceeding with HA rule retrieval.")
|
||||
|
||||
for rule in proxmox_api.cluster.ha.rules.get():
|
||||
|
||||
# Skip disabled rules (disable key exists AND is truthy)
|
||||
if rule.get("disable", 0):
|
||||
logger.debug(f"Skipping ha-rule: {rule['rule']} of type {rule['type']} affecting guests: {rule['resources']}. Rule is disabled.")
|
||||
continue
|
||||
|
||||
# Create a resource list by splitting on commas and stripping whitespace containing
|
||||
# the VM and CT IDs that are part of this HA rule
|
||||
resources_list_guests = [int(r.split(":")[1]) for r in rule["resources"].split(",") if r.strip()]
|
||||
|
||||
# Convert the affinity field to a more descriptive type
|
||||
if rule.get("affinity", None) == "negative":
|
||||
affinity_type = "anti-affinity"
|
||||
else:
|
||||
affinity_type = "affinity"
|
||||
|
||||
# Create affected nodes list
|
||||
resources_list_nodes = []
|
||||
if rule.get("nodes", None):
|
||||
resources_list_nodes = [n for n in rule["nodes"].split(",") if n]
|
||||
|
||||
# Create the ha_rule element
|
||||
ha_rules['ha_rules'][rule['rule']] = {}
|
||||
ha_rules['ha_rules'][rule['rule']]['rule'] = rule['rule']
|
||||
ha_rules['ha_rules'][rule['rule']]['type'] = affinity_type
|
||||
ha_rules['ha_rules'][rule['rule']]['nodes'] = resources_list_nodes
|
||||
ha_rules['ha_rules'][rule['rule']]['members'] = resources_list_guests
|
||||
|
||||
logger.debug(f"Got ha-rule: {rule['rule']} as type {affinity_type} affecting guests: {rule['resources']}")
|
||||
|
||||
logger.debug("Finished: ha_rules.")
|
||||
return ha_rules
|
||||
|
||||
@staticmethod
|
||||
def get_ha_rules_for_guest(guest_name: str, ha_rules: Dict[str, Any], vm_id: int) -> Dict[str, Any]:
|
||||
"""
|
||||
Return the list of HA rules that include the given guest.
|
||||
|
||||
Args:
|
||||
guest_name (str): Name of the VM or CT to look up.
|
||||
ha_rules (Dict[str, Any]): HA rules structure as returned by get_ha_rules(),
|
||||
expected to contain a top-level "ha_rules" mapping each rule id to
|
||||
{"rule": <rule_id>, "type": <affinity_type>, "members": [<resource_ids>...]}.
|
||||
vm_id (int): VM or CT ID of the guest.
|
||||
|
||||
Returns:
|
||||
list: IDs of HA rules the guest is a member of (empty list if none).
|
||||
"""
|
||||
logger.debug("Starting: get_ha_rules_for_guest.")
|
||||
guest_ha_rules = []
|
||||
|
||||
for rule in ha_rules["ha_rules"].values():
|
||||
if vm_id in rule.get("members", []):
|
||||
logger.debug(f"Guest: {guest_name} (VMID: {vm_id}) is member of HA Rule: {rule['rule']}.")
|
||||
guest_ha_rules.append(rule)
|
||||
else:
|
||||
logger.debug(f"Guest: {guest_name} (VMID: {vm_id}) is NOT member of HA Rule: {rule['rule']}.")
|
||||
|
||||
logger.debug("Finished: get_ha_rules_for_guest.")
|
||||
return guest_ha_rules
|
||||
@@ -21,8 +21,10 @@ __copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
|
||||
__license__ = "GPL-3.0"
|
||||
|
||||
|
||||
import time
|
||||
from typing import Dict, Any
|
||||
from utils.logger import SystemdLogger
|
||||
from utils.helper import Helper
|
||||
|
||||
logger = SystemdLogger()
|
||||
|
||||
@@ -47,6 +49,7 @@ class Nodes:
|
||||
|
||||
Args:
|
||||
proxmox_api (any): The Proxmox API client instance.
|
||||
proxmox_config (Dict[str, Any]): A dictionary containing the ProxLB configuration.
|
||||
nodes (Dict[str, Any]): A dictionary containing information about the nodes in the Proxmox cluster.
|
||||
|
||||
Returns:
|
||||
@@ -60,6 +63,8 @@ class Nodes:
|
||||
if node["status"] == "online" and not Nodes.set_node_ignore(proxlb_config, node["node"]):
|
||||
nodes["nodes"][node["node"]] = {}
|
||||
nodes["nodes"][node["node"]]["name"] = node["node"]
|
||||
nodes["nodes"][node["node"]]["pve_version"] = Nodes.get_node_pve_version(proxmox_api, node["node"])
|
||||
nodes["nodes"][node["node"]]["pressure_hot"] = False
|
||||
nodes["nodes"][node["node"]]["maintenance"] = False
|
||||
nodes["nodes"][node["node"]]["cpu_total"] = node["maxcpu"]
|
||||
nodes["nodes"][node["node"]]["cpu_assigned"] = 0
|
||||
@@ -68,13 +73,23 @@ class Nodes:
|
||||
nodes["nodes"][node["node"]]["cpu_assigned_percent"] = nodes["nodes"][node["node"]]["cpu_assigned"] / nodes["nodes"][node["node"]]["cpu_total"] * 100
|
||||
nodes["nodes"][node["node"]]["cpu_free_percent"] = nodes["nodes"][node["node"]]["cpu_free"] / node["maxcpu"] * 100
|
||||
nodes["nodes"][node["node"]]["cpu_used_percent"] = nodes["nodes"][node["node"]]["cpu_used"] / node["maxcpu"] * 100
|
||||
nodes["nodes"][node["node"]]["memory_total"] = node["maxmem"]
|
||||
nodes["nodes"][node["node"]]["cpu_pressure_some_percent"] = Nodes.get_node_rrd_data(proxmox_api, node["node"], "cpu", "some")
|
||||
nodes["nodes"][node["node"]]["cpu_pressure_full_percent"] = Nodes.get_node_rrd_data(proxmox_api, node["node"], "cpu", "full")
|
||||
nodes["nodes"][node["node"]]["cpu_pressure_some_spikes_percent"] = Nodes.get_node_rrd_data(proxmox_api, node["node"], "cpu", "some", spikes=True)
|
||||
nodes["nodes"][node["node"]]["cpu_pressure_full_spikes_percent"] = Nodes.get_node_rrd_data(proxmox_api, node["node"], "cpu", "full", spikes=True)
|
||||
nodes["nodes"][node["node"]]["cpu_pressure_hot"] = False
|
||||
nodes["nodes"][node["node"]]["memory_total"] = Nodes.set_node_resource_reservation(node["node"], node["maxmem"], proxlb_config, "memory")
|
||||
nodes["nodes"][node["node"]]["memory_assigned"] = 0
|
||||
nodes["nodes"][node["node"]]["memory_used"] = node["mem"]
|
||||
nodes["nodes"][node["node"]]["memory_free"] = node["maxmem"] - node["mem"]
|
||||
nodes["nodes"][node["node"]]["memory_assigned_percent"] = nodes["nodes"][node["node"]]["memory_assigned"] / nodes["nodes"][node["node"]]["memory_total"] * 100
|
||||
nodes["nodes"][node["node"]]["memory_free_percent"] = nodes["nodes"][node["node"]]["memory_free"] / node["maxmem"] * 100
|
||||
nodes["nodes"][node["node"]]["memory_used_percent"] = nodes["nodes"][node["node"]]["memory_used"] / node["maxmem"] * 100
|
||||
nodes["nodes"][node["node"]]["memory_pressure_some_percent"] = Nodes.get_node_rrd_data(proxmox_api, node["node"], "memory", "some")
|
||||
nodes["nodes"][node["node"]]["memory_pressure_full_percent"] = Nodes.get_node_rrd_data(proxmox_api, node["node"], "memory", "full")
|
||||
nodes["nodes"][node["node"]]["memory_pressure_some_spikes_percent"] = Nodes.get_node_rrd_data(proxmox_api, node["node"], "memory", "some", spikes=True)
|
||||
nodes["nodes"][node["node"]]["memory_pressure_full_spikes_percent"] = Nodes.get_node_rrd_data(proxmox_api, node["node"], "memory", "full", spikes=True)
|
||||
nodes["nodes"][node["node"]]["memory_pressure_hot"] = False
|
||||
nodes["nodes"][node["node"]]["disk_total"] = node["maxdisk"]
|
||||
nodes["nodes"][node["node"]]["disk_assigned"] = 0
|
||||
nodes["nodes"][node["node"]]["disk_used"] = node["disk"]
|
||||
@@ -82,11 +97,17 @@ class Nodes:
|
||||
nodes["nodes"][node["node"]]["disk_assigned_percent"] = nodes["nodes"][node["node"]]["disk_assigned"] / nodes["nodes"][node["node"]]["disk_total"] * 100
|
||||
nodes["nodes"][node["node"]]["disk_free_percent"] = nodes["nodes"][node["node"]]["disk_free"] / node["maxdisk"] * 100
|
||||
nodes["nodes"][node["node"]]["disk_used_percent"] = nodes["nodes"][node["node"]]["disk_used"] / node["maxdisk"] * 100
|
||||
nodes["nodes"][node["node"]]["disk_pressure_some_percent"] = Nodes.get_node_rrd_data(proxmox_api, node["node"], "disk", "some")
|
||||
nodes["nodes"][node["node"]]["disk_pressure_full_percent"] = Nodes.get_node_rrd_data(proxmox_api, node["node"], "disk", "full")
|
||||
nodes["nodes"][node["node"]]["disk_pressure_some_spikes_percent"] = Nodes.get_node_rrd_data(proxmox_api, node["node"], "disk", "some", spikes=True)
|
||||
nodes["nodes"][node["node"]]["disk_pressure_full_spikes_percent"] = Nodes.get_node_rrd_data(proxmox_api, node["node"], "disk", "full", spikes=True)
|
||||
nodes["nodes"][node["node"]]["disk_pressure_hot"] = False
|
||||
|
||||
# Evaluate if node should be set to maintenance mode
|
||||
if Nodes.set_node_maintenance(proxmox_api, proxlb_config, node["node"]):
|
||||
nodes["nodes"][node["node"]]["maintenance"] = True
|
||||
|
||||
logger.debug(f"Node metrics collected: {nodes}")
|
||||
logger.debug("Finished: get_nodes.")
|
||||
return nodes
|
||||
|
||||
@@ -153,3 +174,139 @@ class Nodes:
|
||||
return True
|
||||
|
||||
logger.debug("Finished: set_node_ignore.")
|
||||
|
||||
@staticmethod
|
||||
def get_node_rrd_data(proxmox_api, node_name: str, object_name: str, object_type: str, spikes=False) -> float:
|
||||
"""
|
||||
Retrieves the rrd data metrics for a specific resource (CPU, memory, disk) of a node.
|
||||
|
||||
Args:
|
||||
proxmox_api (Any): The Proxmox API client instance.
|
||||
node_name (str): The name of the node hosting the guest.
|
||||
object_name (str): The resource type to query (e.g., 'cpu', 'memory', 'disk').
|
||||
object_type (str, optional): The pressure type ('some', 'full') or None for average usage.
|
||||
spikes (bool, optional): Whether to consider spikes in the calculation. Defaults to False.
|
||||
|
||||
Returns:
|
||||
float: The calculated average usage value for the specified resource.
|
||||
"""
|
||||
logger.debug("Starting: get_node_rrd_data.")
|
||||
time.sleep(0.1)
|
||||
|
||||
try:
|
||||
if spikes:
|
||||
logger.debug(f"Getting spike RRD data for {object_name} from node: {node_name}.")
|
||||
node_data_rrd = proxmox_api.nodes(node_name).rrddata.get(timeframe="hour", cf="MAX")
|
||||
else:
|
||||
logger.debug(f"Getting average RRD data for {object_name} from node: {node_name}.")
|
||||
node_data_rrd = proxmox_api.nodes(node_name).rrddata.get(timeframe="hour", cf="AVERAGE")
|
||||
|
||||
except Exception:
|
||||
logger.error(f"Failed to retrieve RRD data for guest: {node_name}. Using 0.0 as value.")
|
||||
logger.debug("Finished: get_node_rrd_data.")
|
||||
return 0.0
|
||||
|
||||
lookup_key = f"pressure{object_name}{object_type}"
|
||||
|
||||
if spikes:
|
||||
# RRD data is collected every minute, so we look at the last 6 entries
|
||||
# and take the maximum value to represent the spike
|
||||
rrd_data_value = [row.get(lookup_key) for row in node_data_rrd if row.get(lookup_key) is not None]
|
||||
rrd_data_value = max(rrd_data_value[-6:], default=0.0)
|
||||
else:
|
||||
# Calculate the average value from the RRD data entries
|
||||
rrd_data_value = sum(entry.get(lookup_key, 0.0) for entry in node_data_rrd) / len(node_data_rrd)
|
||||
|
||||
logger.debug(f"RRD data (spike: {spikes}) for {object_name} from node: {node_name}: {rrd_data_value}")
|
||||
logger.debug("Finished: get_node_rrd_data.")
|
||||
return rrd_data_value
|
||||
|
||||
@staticmethod
|
||||
def get_node_pve_version(proxmox_api, node_name: str) -> float:
|
||||
"""
|
||||
Return the Proxmox VE (PVE) version for a given node by querying the Proxmox API.
|
||||
|
||||
This function calls proxmox_api.nodes(node_name).version.get() and extracts the
|
||||
'version' field from the returned mapping. The value is expected to be numeric
|
||||
(or convertible to float) and is returned as a float.
|
||||
|
||||
Args:
|
||||
proxmox_api (Any): The Proxmox API client instance.
|
||||
node_name (str): The name of the node hosting the guest.
|
||||
|
||||
Returns:
|
||||
float: The PVE version for the specified node as a floating point number.
|
||||
|
||||
Raises:
|
||||
Exception: If the proxmox_api call fails, returns an unexpected structure, or the
|
||||
'version' field is missing or cannot be converted to float. Callers should
|
||||
handle or propagate exceptions as appropriate.
|
||||
"""
|
||||
logger.debug("Starting: get_node_pve_version.")
|
||||
time.sleep(0.1)
|
||||
|
||||
try:
|
||||
logger.debug(f"Trying to get PVE version for node: {node_name}.")
|
||||
version = proxmox_api.nodes(node_name).version.get()
|
||||
except Exception:
|
||||
logger.error(f"Failed to get PVE version for node: {node_name}.")
|
||||
|
||||
logger.debug(f"Got version {version['version']} for node {node_name}.")
|
||||
logger.debug("Finished: get_node_pve_version.")
|
||||
return version["version"]
|
||||
|
||||
@staticmethod
|
||||
def set_node_resource_reservation(node_name, resource_value, proxlb_config, resource_type) -> int:
|
||||
"""
|
||||
Check if there is a configured resource reservation for the current node and apply it as needed.
|
||||
Checks for a node specific config first, then if there is any configured default and if neither then nothing is reserved.
|
||||
Reservations are applied by directly modifying the resource value.
|
||||
|
||||
Args:
|
||||
node_name (str): The name of the node.
|
||||
resource_value (int): The total resource value in bytes.
|
||||
proxlb_config (Dict[str, Any]): A dictionary containing the ProxLB configuration.
|
||||
resource_type (str): The type of resource ('memory', 'disk', etc.).
|
||||
|
||||
Returns:
|
||||
int: The resource value after applying any configured reservations.
|
||||
"""
|
||||
logger.debug(f"Starting: apply_resource_reservation")
|
||||
|
||||
balancing_cfg = proxlb_config.get("balancing", {})
|
||||
reserve_cfg = balancing_cfg.get("node_resource_reserve", {})
|
||||
node_resource_reservation = reserve_cfg.get(node_name, {}).get(resource_type, 0)
|
||||
default_resource_reservation = reserve_cfg.get("defaults", {}).get(resource_type, 0)
|
||||
|
||||
# Ensure reservations are numeric values
|
||||
node_resource_reservation = node_resource_reservation if isinstance(node_resource_reservation, (int, float)) else 0
|
||||
default_resource_reservation = default_resource_reservation if isinstance(default_resource_reservation, (int, float)) else 0
|
||||
|
||||
# Apply node specific reservation if set
|
||||
if node_resource_reservation > 0:
|
||||
if resource_value < (node_resource_reservation * 1024 ** 3):
|
||||
logger.critical(f"Configured resource reservation for node {node_name} of type {resource_type} with {node_resource_reservation} GB is higher than available resource value {resource_value / (1024 ** 3):.2f} GB. Not applying...")
|
||||
return resource_value
|
||||
else:
|
||||
logger.debug(f"Applying node specific reservation for {node_name} of type {resource_type} with {node_resource_reservation} GB.")
|
||||
resource_value_new = resource_value - (node_resource_reservation * 1024 ** 3)
|
||||
logger.debug(f'Switched resource value for node {node_name} of type {resource_type} from {resource_value / (1024 ** 3):.2f} GB to {resource_value_new / (1024 ** 3):.2f} GB after applying reservation.')
|
||||
logger.debug(f"Before: {resource_value} | After: {resource_value_new}")
|
||||
return resource_value_new
|
||||
|
||||
# Apply default reservation if set and no node specific reservation has been performed
|
||||
elif default_resource_reservation > 0:
|
||||
if resource_value < (default_resource_reservation * 1024 ** 3):
|
||||
logger.critical(f"Configured default reservation for node {node_name} of type {resource_type} with {default_resource_reservation} GB is higher than available resource value {resource_value / (1024 ** 3):.2f} GB. Not applying...")
|
||||
return resource_value
|
||||
else:
|
||||
logger.debug(f"Applying default reservation for {node_name} of type {resource_type} with {default_resource_reservation} GB.")
|
||||
resource_value_new = resource_value - (default_resource_reservation * 1024 ** 3)
|
||||
logger.debug(f'Switched resource value for node {node_name} of type {resource_type} from {resource_value / (1024 ** 3):.2f} GB to {resource_value_new / (1024 ** 3):.2f} GB after applying reservation.')
|
||||
logger.debug(f"Before: {resource_value} | After: {resource_value_new}")
|
||||
return resource_value_new
|
||||
|
||||
else:
|
||||
logger.debug(f"No default or node specific resource reservation for node {node_name} found. Skipping...")
|
||||
logger.debug(f"Finished: apply_resource_reservation")
|
||||
return resource_value
|
||||
|
||||
143
proxlb/models/pools.py
Normal file
143
proxlb/models/pools.py
Normal file
@@ -0,0 +1,143 @@
|
||||
"""
|
||||
The Pools class retrieves all present pools defined on a Proxmox cluster
|
||||
including the chield objects.
|
||||
"""
|
||||
|
||||
__author__ = "Florian Paul Azim Hoberg <gyptazy>"
|
||||
__copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
|
||||
__license__ = "GPL-3.0"
|
||||
|
||||
|
||||
from typing import Dict, Any
|
||||
from utils.logger import SystemdLogger
|
||||
from models.tags import Tags
|
||||
import time
|
||||
|
||||
logger = SystemdLogger()
|
||||
|
||||
|
||||
class Pools:
|
||||
"""
|
||||
The Pools class retrieves all present pools defined on a Proxmox cluster
|
||||
including the chield objects.
|
||||
|
||||
Methods:
|
||||
__init__:
|
||||
Initializes the Pools class.
|
||||
|
||||
get_pools(proxmox_api: any) -> Dict[str, Any]:
|
||||
Retrieve pool definitions and membership from the Proxmox cluster.
|
||||
Returns a dict with a top-level "pools" mapping each poolid to
|
||||
{"name": <poolid>, "members": [<member_names>...]}.
|
||||
This method does not collect per-member metrics or perform node filtering.
|
||||
"""
|
||||
def __init__(self):
|
||||
"""
|
||||
Initializes the Pools class with the provided ProxLB data.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_pools(proxmox_api: any) -> Dict[str, Any]:
|
||||
"""
|
||||
Retrieve all pools and their members from a Proxmox cluster.
|
||||
|
||||
Queries the Proxmox API for pool definitions and returns a dictionary
|
||||
containing each pool's id/name and a list of its member VM/CT names.
|
||||
This function does not perform per-member metric collection or node
|
||||
filtering — it only gathers pool membership information.
|
||||
|
||||
Args:
|
||||
proxmox_api (any): Proxmox API client instance.
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: Dictionary with a top-level "pools" key mapping poolid
|
||||
to {"name": <poolid>, "members": [<member_names>...]}.
|
||||
"""
|
||||
logger.debug("Starting: get_pools.")
|
||||
pools = {"pools": {}}
|
||||
|
||||
# Pool objects: iterate over all pools in the cluster.
|
||||
# We keep pool members even if their nodes are ignored so resource accounting
|
||||
# for rebalancing remains correct and we avoid overprovisioning nodes.
|
||||
for pool in proxmox_api.pools.get():
|
||||
logger.debug(f"Got pool: {pool['poolid']}")
|
||||
pools['pools'][pool['poolid']] = {}
|
||||
pools['pools'][pool['poolid']]['name'] = pool['poolid']
|
||||
pools['pools'][pool['poolid']]['members'] = []
|
||||
|
||||
# Fetch pool details and collect member names
|
||||
pool_details = proxmox_api.pools(pool['poolid']).get()
|
||||
for member in pool_details.get("members", []):
|
||||
|
||||
# We might also have objects without the key "name", e.g. storage pools
|
||||
if "name" not in member:
|
||||
logger.debug(f"Skipping member without name in pool: {pool['poolid']}")
|
||||
continue
|
||||
|
||||
logger.debug(f"Got member: {member['name']} for pool: {pool['poolid']}")
|
||||
pools['pools'][pool['poolid']]['members'].append(member["name"])
|
||||
|
||||
logger.debug("Finished: get_pools.")
|
||||
return pools
|
||||
|
||||
@staticmethod
|
||||
def get_pools_for_guest(guest_name: str, pools: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Return the list of pool names that include the given guest.
|
||||
|
||||
Args:
|
||||
guest_name (str): Name of the VM or CT to look up.
|
||||
pools (Dict[str, Any]): Pools structure as returned by get_pools(),
|
||||
expected to contain a top-level "pools" mapping each poolid to
|
||||
{"name": <poolid>, "members": [<member_names>...]}.
|
||||
|
||||
Returns:
|
||||
list[str]: Names of pools the guest is a member of (empty list if none).
|
||||
"""
|
||||
logger.debug("Starting: get_pools_for_guests.")
|
||||
guest_pools = []
|
||||
|
||||
for pool in pools.items():
|
||||
for pool_id, pool_data in pool[1].items():
|
||||
|
||||
if type(pool_data) is dict:
|
||||
pool_name = pool_data.get("name", "")
|
||||
pool_name_members = pool_data.get("members", [])
|
||||
|
||||
if guest_name in pool_name_members:
|
||||
logger.debug(f"Guest: {guest_name} is member of Pool: {pool_name}.")
|
||||
guest_pools.append(pool_name)
|
||||
else:
|
||||
logger.debug(f"Guest: {guest_name} is NOT member of Pool: {pool_name}.")
|
||||
|
||||
else:
|
||||
logger.debug(f"Pool data for pool_id {pool_id} is not a dict: {pool_data}")
|
||||
|
||||
logger.debug("Finished: get_pools_for_guests.")
|
||||
return guest_pools
|
||||
|
||||
@staticmethod
|
||||
def get_pool_node_affinity_strictness(proxlb_config: Dict[str, Any], guest_pools: list) -> bool:
|
||||
"""
|
||||
Retrieve the node affinity strictness setting for a guest across its pools.
|
||||
|
||||
Queries the ProxLB configuration to determine the node affinity strictness
|
||||
level for the specified guest based on its pool memberships. Returns the
|
||||
strictness setting from the first matching pool configuration.
|
||||
|
||||
Args:
|
||||
proxlb_config (Dict[str, Any]): ProxLB configuration dictionary.
|
||||
guest_pools (list): List of pool names the guest belongs to.
|
||||
|
||||
Returns:
|
||||
bool: Node affinity strictness setting (default True if not specified).
|
||||
"""
|
||||
logger.debug("Starting: get_pool_node_affinity_strictness.")
|
||||
|
||||
node_strictness = True
|
||||
for pool in guest_pools:
|
||||
pool_settings = proxlb_config.get("balancing", {}).get("pools", {}).get(pool, {})
|
||||
node_strictness = pool_settings.get("strict", True)
|
||||
|
||||
logger.debug("Finished: get_pool_node_affinity_strictness.")
|
||||
return node_strictness
|
||||
@@ -12,7 +12,9 @@ __license__ = "GPL-3.0"
|
||||
|
||||
import time
|
||||
from typing import List
|
||||
from typing import Dict, Any
|
||||
from utils.logger import SystemdLogger
|
||||
from utils.helper import Helper
|
||||
|
||||
logger = SystemdLogger()
|
||||
|
||||
@@ -78,15 +80,19 @@ class Tags:
|
||||
return tags
|
||||
|
||||
@staticmethod
|
||||
def get_affinity_groups(tags: List[str]) -> List[str]:
|
||||
def get_affinity_groups(tags: List[str], pools: List[str], ha_rules: List[str], proxlb_config: Dict[str, Any]) -> List[str]:
|
||||
"""
|
||||
Get affinity tags for a guest from the Proxmox cluster by the API.
|
||||
|
||||
This method retrieves all tags for a given guest and evaluates the
|
||||
affinity tags which are required during the balancing calculations.
|
||||
This method retrieves all tags for a given guest or based on a
|
||||
membership of a pool and evaluates the affinity groups which are
|
||||
required during the balancing calculations.
|
||||
|
||||
Args:
|
||||
tags (List): A list holding all defined tags for a given guest.
|
||||
pools (List): A list holding all defined pools for a given guest.
|
||||
ha_rules (List): A list holding all defined ha_rules for a given guest.
|
||||
proxlb_config (Dict): A dict holding the ProxLB configuration.
|
||||
|
||||
Returns:
|
||||
List: A list including all affinity tags for the given guest.
|
||||
@@ -94,24 +100,49 @@ class Tags:
|
||||
logger.debug("Starting: get_affinity_groups.")
|
||||
affinity_tags = []
|
||||
|
||||
# Tag based affinity groups
|
||||
if len(tags) > 0:
|
||||
for tag in tags:
|
||||
if tag.startswith("plb_affinity"):
|
||||
logger.debug(f"Adding affinity group for tag {tag}.")
|
||||
affinity_tags.append(tag)
|
||||
else:
|
||||
logger.debug(f"Skipping evaluation of tag: {tag} This is not an affinity tag.")
|
||||
|
||||
# Pool based affinity groups
|
||||
if len(pools) > 0:
|
||||
for pool in pools:
|
||||
if pool in (proxlb_config['balancing'].get('pools') or {}):
|
||||
if proxlb_config['balancing']['pools'][pool].get('type', None) == 'affinity':
|
||||
logger.debug(f"Adding affinity group for pool {pool}.")
|
||||
affinity_tags.append(pool)
|
||||
else:
|
||||
logger.debug(f"Skipping evaluation of pool: {pool} This is not an affinity pool.")
|
||||
|
||||
# HA rule based affinity groups
|
||||
if len(ha_rules) > 0:
|
||||
for ha_rule in ha_rules:
|
||||
if ha_rule.get('type', None) == 'affinity':
|
||||
logger.debug(f"Adding affinity group for ha-rule {ha_rule}.")
|
||||
affinity_tags.append(ha_rule['rule'])
|
||||
|
||||
logger.debug("Finished: get_affinity_groups.")
|
||||
return affinity_tags
|
||||
|
||||
@staticmethod
|
||||
def get_anti_affinity_groups(tags: List[str]) -> List[str]:
|
||||
def get_anti_affinity_groups(tags: List[str], pools: List[str], ha_rules: List[str], proxlb_config: Dict[str, Any]) -> List[str]:
|
||||
"""
|
||||
Get anti-affinity tags for a guest from the Proxmox cluster by the API.
|
||||
|
||||
This method retrieves all tags for a given guest and evaluates the
|
||||
anti-affinity tags which are required during the balancing calculations.
|
||||
This method retrieves all tags for a given guest or based on a
|
||||
membership of a pool and evaluates the anti-affinity groups which
|
||||
are required during the balancing calculations.
|
||||
|
||||
Args:
|
||||
tags (List): A list holding all defined tags for a given guest.
|
||||
pools (List): A list holding all defined pools for a given guest.
|
||||
ha_rules (List): A list holding all defined ha_rules for a given guest.
|
||||
proxlb_config (Dict): A dict holding the ProxLB configuration.
|
||||
|
||||
Returns:
|
||||
List: A list including all anti-affinity tags for the given guest..
|
||||
@@ -119,10 +150,31 @@ class Tags:
|
||||
logger.debug("Starting: get_anti_affinity_groups.")
|
||||
anti_affinity_tags = []
|
||||
|
||||
# Tag based anti-affinity groups
|
||||
if len(tags) > 0:
|
||||
for tag in tags:
|
||||
if tag.startswith("plb_anti_affinity"):
|
||||
logger.debug(f"Adding anti-affinity group for tag {tag}.")
|
||||
anti_affinity_tags.append(tag)
|
||||
else:
|
||||
logger.debug(f"Skipping evaluation of tag: {tag} This is not an anti-affinity tag.")
|
||||
|
||||
# Pool based anti-affinity groups
|
||||
if len(pools) > 0:
|
||||
for pool in pools:
|
||||
if pool in (proxlb_config['balancing'].get('pools') or {}):
|
||||
if proxlb_config['balancing']['pools'][pool].get('type', None) == 'anti-affinity':
|
||||
logger.debug(f"Adding anti-affinity group for pool {pool}.")
|
||||
anti_affinity_tags.append(pool)
|
||||
else:
|
||||
logger.debug(f"Skipping evaluation of pool: {pool} This is not an anti-affinity pool.")
|
||||
|
||||
# HA rule based anti-affinity groups
|
||||
if len(ha_rules) > 0:
|
||||
for ha_rule in ha_rules:
|
||||
if ha_rule.get('type', None) == 'anti-affinity':
|
||||
logger.debug(f"Adding anti-affinity group for ha-rule {ha_rule}.")
|
||||
anti_affinity_tags.append(ha_rule['rule'])
|
||||
|
||||
logger.debug("Finished: get_anti_affinity_groups.")
|
||||
return anti_affinity_tags
|
||||
@@ -145,36 +197,87 @@ class Tags:
|
||||
ignore_tag = False
|
||||
|
||||
if len(tags) > 0:
|
||||
logger.debug(f"Found {len(tags)} tags to evaluate.")
|
||||
for tag in tags:
|
||||
logger.debug(f"Evaluating tag: {tag}.")
|
||||
if tag.startswith("plb_ignore"):
|
||||
logger.debug(f"Found valid ignore tag: {tag}. Marking guest as ignored.")
|
||||
ignore_tag = True
|
||||
else:
|
||||
logger.debug(f"Tag: {tag} This is not an ignore tag.")
|
||||
|
||||
logger.debug("Finished: get_ignore.")
|
||||
return ignore_tag
|
||||
|
||||
@staticmethod
|
||||
def get_node_relationships(tags: List[str]) -> str:
|
||||
def get_node_relationships(tags: List[str], nodes: Dict[str, Any], pools: List[str], ha_rules: List[str], proxlb_config: Dict[str, Any]) -> str:
|
||||
"""
|
||||
Get a node relationship tag for a guest from the Proxmox cluster by the API to pin
|
||||
a guest to a node.
|
||||
a guest to a node or by defined pools from ProxLB configuration.
|
||||
|
||||
This method retrieves a relationship tag between a guest and a specific
|
||||
hypervisor node to pin the guest to a specific node (e.g., for licensing reason).
|
||||
|
||||
Args:
|
||||
tags (List): A list holding all defined tags for a given guest.
|
||||
nodes (Dict): A dictionary holding all available nodes in the cluster.
|
||||
pools (List): A list holding all defined pools for a given guest.
|
||||
ha_rules (List): A list holding all defined ha_rules for a given guest.
|
||||
proxlb_config (Dict): A dict holding the ProxLB configuration.
|
||||
|
||||
Returns:
|
||||
Str: The related hypervisor node name.
|
||||
Str: The related hypervisor node name(s).
|
||||
"""
|
||||
logger.debug("Starting: get_node_relationships.")
|
||||
node_relationship_tags = []
|
||||
|
||||
# Tag based node relationship
|
||||
if len(tags) > 0:
|
||||
logger.debug("Validating node pinning by tags.")
|
||||
for tag in tags:
|
||||
if tag.startswith("plb_pin"):
|
||||
node_relationship_tag = tag.replace("plb_pin_", "")
|
||||
node_relationship_tags.append(node_relationship_tag)
|
||||
|
||||
# Validate if the node to pin is present in the cluster
|
||||
if Helper.validate_node_presence(node_relationship_tag, nodes):
|
||||
logger.debug(f"Tag {node_relationship_tag} is valid! Defined node exists in the cluster.")
|
||||
logger.debug(f"Setting node relationship because of tag {tag} to {node_relationship_tag}.")
|
||||
node_relationship_tags.append(node_relationship_tag)
|
||||
else:
|
||||
logger.warning(f"Tag {node_relationship_tag} is invalid! Defined node does not exist in the cluster. Not applying pinning.")
|
||||
|
||||
# Pool based node relationship
|
||||
if len(pools) > 0:
|
||||
logger.debug("Validating node pinning by pools.")
|
||||
for pool in pools:
|
||||
if pool in (proxlb_config['balancing'].get('pools') or {}):
|
||||
|
||||
pool_nodes = proxlb_config['balancing']['pools'][pool].get('pin', None)
|
||||
for node in pool_nodes if pool_nodes is not None else []:
|
||||
|
||||
# Validate if the node to pin is present in the cluster
|
||||
if Helper.validate_node_presence(node, nodes):
|
||||
logger.debug(f"Pool pinning tag {node} is valid! Defined node exists in the cluster.")
|
||||
logger.debug(f"Setting node relationship because of pool {pool} to {node}.")
|
||||
node_relationship_tags.append(node)
|
||||
else:
|
||||
logger.warning(f"Pool pinning tag {node} is invalid! Defined node does not exist in the cluster. Not applying pinning.")
|
||||
|
||||
else:
|
||||
logger.debug(f"Skipping pinning for pool {pool}. Pool is not defined in ProxLB configuration.")
|
||||
|
||||
# HA rule based node relationship
|
||||
if len(ha_rules) > 0:
|
||||
logger.debug("Validating node pinning by ha-rules.")
|
||||
for ha_rule in ha_rules:
|
||||
if len(ha_rule.get("nodes", 0)) > 0:
|
||||
if ha_rule.get("type", None) == "affinity":
|
||||
logger.debug(f"ha-rule {ha_rule['rule']} is of type affinity.")
|
||||
for node in ha_rule["nodes"]:
|
||||
logger.debug(f"Adding {node} as node relationship because of ha-rule {ha_rule['rule']}.")
|
||||
node_relationship_tags.append(node)
|
||||
else:
|
||||
logger.debug(f"ha-rule {ha_rule['rule']} is of type anti-affinity. Skipping node relationship addition.")
|
||||
|
||||
logger.debug("Finished: get_node_relationships.")
|
||||
return node_relationship_tags
|
||||
|
||||
@@ -11,11 +11,13 @@ __license__ = "GPL-3.0"
|
||||
import json
|
||||
import uuid
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import utils.version
|
||||
from utils.logger import SystemdLogger
|
||||
from typing import Dict, Any
|
||||
from types import FrameType
|
||||
|
||||
logger = SystemdLogger()
|
||||
|
||||
@@ -79,6 +81,7 @@ class Helper:
|
||||
"""
|
||||
logger.debug("Starting: log_node_metrics.")
|
||||
nodes_usage_memory = " | ".join([f"{key}: {value['memory_used_percent']:.2f}%" for key, value in proxlb_data["nodes"].items()])
|
||||
nodes_assigned_memory = " | ".join([f"{key}: {value['memory_assigned_percent']:.2f}%" for key, value in proxlb_data["nodes"].items()])
|
||||
nodes_usage_cpu = " | ".join([f"{key}: {value['cpu_used_percent']:.2f}%" for key, value in proxlb_data["nodes"].items()])
|
||||
nodes_usage_disk = " | ".join([f"{key}: {value['disk_used_percent']:.2f}%" for key, value in proxlb_data["nodes"].items()])
|
||||
|
||||
@@ -88,6 +91,7 @@ class Helper:
|
||||
proxlb_data["meta"]["statistics"]["after"] = {"memory": nodes_usage_memory, "cpu": nodes_usage_cpu, "disk": nodes_usage_disk}
|
||||
|
||||
logger.debug(f"Nodes usage memory: {nodes_usage_memory}")
|
||||
logger.debug(f"Nodes usage memory assigned: {nodes_assigned_memory}")
|
||||
logger.debug(f"Nodes usage cpu: {nodes_usage_cpu}")
|
||||
logger.debug(f"Nodes usage disk: {nodes_usage_disk}")
|
||||
logger.debug("Finished: log_node_metrics.")
|
||||
@@ -200,7 +204,7 @@ class Helper:
|
||||
logger.debug("Finished: print_json.")
|
||||
|
||||
@staticmethod
|
||||
def handler_sighup(signum, frame):
|
||||
def handler_sighup(signum: int, frame: FrameType) -> None:
|
||||
"""
|
||||
Signal handler for SIGHUP.
|
||||
|
||||
@@ -217,6 +221,23 @@ class Helper:
|
||||
Helper.proxlb_reload = True
|
||||
logger.debug("Finished: handle_sighup.")
|
||||
|
||||
@staticmethod
|
||||
def handler_sigint(signum: int, frame: FrameType) -> None:
|
||||
"""
|
||||
Signal handler for SIGINT. (triggered by CTRL+C).
|
||||
|
||||
Args:
|
||||
signum (int): The signal number (e.g., SIGINT).
|
||||
frame (FrameType): The current stack frame when the signal was received.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
exit_message = "ProxLB has been successfully terminated by user."
|
||||
logger.debug(exit_message)
|
||||
print(f"\n {exit_message}")
|
||||
sys.exit(0)
|
||||
|
||||
@staticmethod
|
||||
def get_host_port_from_string(host_object):
|
||||
"""
|
||||
@@ -265,3 +286,52 @@ class Helper:
|
||||
return parts[0], port
|
||||
except ValueError:
|
||||
return host_object, 8006
|
||||
|
||||
@staticmethod
|
||||
def validate_node_presence(node: str, nodes: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
Validates whether a given node exists in the provided cluster nodes dictionary.
|
||||
|
||||
Args:
|
||||
node (str): The name of the node to validate.
|
||||
nodes (Dict[str, Any]): A dictionary containing cluster information.
|
||||
Must include a "nodes" key mapping to a dict of available nodes.
|
||||
|
||||
Returns:
|
||||
bool: True if the node exists in the cluster, False otherwise.
|
||||
"""
|
||||
logger.debug("Starting: validate_node_presence.")
|
||||
|
||||
if node in nodes["nodes"].keys():
|
||||
logger.info(f"Node {node} found in cluster. Applying pinning.")
|
||||
logger.debug("Finished: validate_node_presence.")
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"Node {node} not found in cluster. Not applying pinning!")
|
||||
logger.debug("Finished: validate_node_presence.")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def tcp_connect_test(addr_family: int, host: str, port: int, timeout: int) -> tuple[bool, int | None]:
|
||||
"""
|
||||
Attempt a TCP connection to the specified host and port to test the reachability.
|
||||
|
||||
Args:
|
||||
addr_family (int): Address family for the socket (e.g., socket.AF_INET for IPv4, socket.AF_INET6 for IPv6).
|
||||
host (str): The hostname or IP address to connect to.
|
||||
port (int): The port number to connect to.
|
||||
timeout (int): Connection timeout in seconds.
|
||||
|
||||
Returns:
|
||||
tuple[bool, int | None]: A tuple containing:
|
||||
- bool: True if the connection was successful, False otherwise.
|
||||
- int | None: None if the connection was successful, otherwise the errno code indicating the reason for failure.
|
||||
"""
|
||||
test_socket = socket.socket(addr_family, socket.SOCK_STREAM)
|
||||
test_socket.settimeout(timeout)
|
||||
|
||||
try:
|
||||
rc = test_socket.connect_ex((host, port))
|
||||
return (rc == 0, rc if rc != 0 else None)
|
||||
finally:
|
||||
test_socket.close()
|
||||
|
||||
@@ -88,7 +88,7 @@ class SystemdLogger:
|
||||
# logging is preferred.
|
||||
if SYSTEMD_PRESENT:
|
||||
# Add a JournalHandler for systemd integration
|
||||
handler = JournalHandler()
|
||||
handler = JournalHandler(SYSLOG_IDENTIFIER="ProxLB")
|
||||
else:
|
||||
# Add a stdout handler as a fallback
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
|
||||
@@ -13,6 +13,7 @@ __copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
|
||||
__license__ = "GPL-3.0"
|
||||
|
||||
|
||||
import errno
|
||||
try:
|
||||
import proxmoxer
|
||||
PROXMOXER_PRESENT = True
|
||||
@@ -135,6 +136,14 @@ class ProxmoxApi:
|
||||
proxlb_credentials = proxlb_config["proxmox_api"]
|
||||
present_auth_pass = "pass" in proxlb_credentials
|
||||
present_auth_secret = "token_secret" in proxlb_credentials
|
||||
token_id = proxlb_credentials.get("token_id", None)
|
||||
|
||||
if token_id:
|
||||
non_allowed_chars = ["@", "!"]
|
||||
for char in non_allowed_chars:
|
||||
if char in token_id:
|
||||
logger.error(f"Wrong user/token format defined. User and token id must be splitted! Please see: https://github.com/gyptazy/ProxLB/blob/main/docs/03_configuration.md#required-permissions-for-a-user")
|
||||
sys.exit(1)
|
||||
|
||||
if present_auth_pass and present_auth_secret:
|
||||
logger.critical(f"Username/password and API token authentication are mutal exclusive. Please use only one!")
|
||||
@@ -167,49 +176,40 @@ class ProxmoxApi:
|
||||
logger.debug("Starting: api_connect_get_hosts.")
|
||||
# Pre-validate the given API endpoints
|
||||
if not isinstance(proxmox_api_endpoints, list):
|
||||
logger.critical(f"The proxmox_api hosts are not defined as a list type.")
|
||||
logger.critical("The proxmox_api hosts are not defined as a list type.")
|
||||
sys.exit(1)
|
||||
|
||||
if not proxmox_api_endpoints:
|
||||
logger.critical(f"No proxmox_api hosts are defined.")
|
||||
logger.critical("No proxmox_api hosts are defined.")
|
||||
sys.exit(1)
|
||||
|
||||
if len(proxmox_api_endpoints) == 0:
|
||||
logger.critical(f"No proxmox_api hosts are defined.")
|
||||
sys.exit(1)
|
||||
validated_api_hosts: list[tuple[str, int]] = []
|
||||
|
||||
# If we have multiple Proxmox API endpoints, we need to check each one by
|
||||
# doing a connection attempt for IPv4 and IPv6. If we find a working one,
|
||||
# we return that one. This allows us to define multiple endpoints in a cluster.
|
||||
validated_api_hosts = []
|
||||
for host in proxmox_api_endpoints:
|
||||
retries = proxlb_config["proxmox_api"].get("retries", 1)
|
||||
wait_time = proxlb_config["proxmox_api"].get("wait_time", 1)
|
||||
|
||||
# Get or set a default value for a maximum of retries when connecting to
|
||||
# the Proxmox API
|
||||
api_connection_retries = proxlb_config["proxmox_api"].get("retries", 1)
|
||||
api_connection_wait_time = proxlb_config["proxmox_api"].get("wait_time", 1)
|
||||
|
||||
for api_connection_attempt in range(api_connection_retries):
|
||||
validated_api_host, api_port = self.test_api_proxmox_host(host)
|
||||
if validated_api_host:
|
||||
validated_api_hosts.append(validated_api_host)
|
||||
for attempt in range(retries):
|
||||
candidate_host, candidate_port = self.test_api_proxmox_host(host)
|
||||
if candidate_host:
|
||||
validated_api_hosts.append((candidate_host, candidate_port))
|
||||
break
|
||||
else:
|
||||
logger.warning(f"Attempt {api_connection_attempt + 1}/{api_connection_retries} failed for host {host}. Retrying in {api_connection_wait_time} seconds...")
|
||||
time.sleep(api_connection_wait_time)
|
||||
logger.warning(
|
||||
f"Attempt {attempt + 1}/{retries} failed for host {host}. "
|
||||
f"Retrying in {wait_time} seconds..."
|
||||
)
|
||||
time.sleep(wait_time)
|
||||
|
||||
if len(validated_api_hosts) > 0:
|
||||
# Choose a random host to distribute the load across the cluster
|
||||
# as a simple load balancing mechanism.
|
||||
return random.choice(validated_api_hosts), api_port
|
||||
if validated_api_hosts:
|
||||
chosen_host, chosen_port = random.choice(validated_api_hosts)
|
||||
return chosen_host, chosen_port
|
||||
|
||||
logger.critical("No valid Proxmox API hosts found.")
|
||||
print("No valid Proxmox API hosts found.")
|
||||
|
||||
logger.debug("Finished: api_connect_get_hosts.")
|
||||
sys.exit(1)
|
||||
|
||||
def test_api_proxmox_host(self, host: str) -> str:
|
||||
def test_api_proxmox_host(self, host: str) -> tuple[str, int | None, None]:
|
||||
"""
|
||||
Tests the connectivity to a Proxmox host by resolving its IP address and
|
||||
checking both IPv4 and IPv6 addresses.
|
||||
@@ -229,31 +229,36 @@ class ProxmoxApi:
|
||||
"""
|
||||
logger.debug("Starting: test_api_proxmox_host.")
|
||||
|
||||
# Validate for custom ports in API hosts which might indicate
|
||||
# that an external loadbalancer will be used.
|
||||
# Validate for custom port configurations (e.g., by given external
|
||||
# loadbalancer systems)
|
||||
host, port = Helper.get_host_port_from_string(host)
|
||||
if port is None:
|
||||
port = 8006
|
||||
|
||||
# Try resolving DNS to IP and log non-resolvable ones
|
||||
try:
|
||||
ip = socket.getaddrinfo(host, None, socket.AF_UNSPEC)
|
||||
infos = socket.getaddrinfo(host, None, socket.AF_UNSPEC)
|
||||
except socket.gaierror:
|
||||
logger.warning(f"Could not resolve {host}.")
|
||||
return False
|
||||
return (None, None)
|
||||
|
||||
# Validate if given object is IPv4 or IPv6
|
||||
for address_type in ip:
|
||||
if address_type[0] == socket.AF_INET:
|
||||
logger.debug(f"{host} is type ipv4.")
|
||||
if self.test_api_proxmox_host_ipv4(host, port):
|
||||
return host, port
|
||||
elif address_type[0] == socket.AF_INET6:
|
||||
logger.debug(f"{host} is type ipv6.")
|
||||
if self.test_api_proxmox_host_ipv6(host, port):
|
||||
return host, port
|
||||
else:
|
||||
return False
|
||||
# Check both families that are actually present
|
||||
saw_family = set()
|
||||
for family, *_rest in infos:
|
||||
saw_family.add(family)
|
||||
|
||||
logger.debug("Finished: test_api_proxmox_host.")
|
||||
if socket.AF_INET in saw_family:
|
||||
logger.debug(f"{host} has IPv4.")
|
||||
if self.test_api_proxmox_host_ipv4(host, port):
|
||||
return (host, port)
|
||||
|
||||
if socket.AF_INET6 in saw_family:
|
||||
logger.debug(f"{host} has IPv6.")
|
||||
if self.test_api_proxmox_host_ipv6(host, port):
|
||||
return (host, port)
|
||||
|
||||
logger.debug("Finished: test_api_proxmox_host (unreachable).")
|
||||
return (None, None)
|
||||
|
||||
def test_api_proxmox_host_ipv4(self, host: str, port: int = 8006, timeout: int = 1) -> bool:
|
||||
"""
|
||||
@@ -272,18 +277,16 @@ class ProxmoxApi:
|
||||
bool: True if the host is reachable on the specified port, False otherwise.
|
||||
"""
|
||||
logger.debug("Starting: test_api_proxmox_host_ipv4.")
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(timeout)
|
||||
logger.warning(f"Warning: Host {host} ran into a timeout when connecting on IPv4 for tcp/{port}.")
|
||||
result = sock.connect_ex((host, port))
|
||||
|
||||
if result == 0:
|
||||
sock.close()
|
||||
ok, rc = Helper.tcp_connect_test(socket.AF_INET, host, port, timeout)
|
||||
if ok:
|
||||
logger.debug(f"Host {host} is reachable on IPv4 for tcp/{port}.")
|
||||
logger.debug("Finished: test_api_proxmox_host_ipv4.")
|
||||
return True
|
||||
|
||||
sock.close()
|
||||
logger.warning(f"Host {host} is unreachable on IPv4 for tcp/{port}.")
|
||||
if rc == errno.ETIMEDOUT:
|
||||
logger.warning(f"Timeout connecting to {host} on IPv4 tcp/{port}.")
|
||||
else:
|
||||
logger.warning(f"Host {host} is unreachable on IPv4 for tcp/{port} (errno {rc}).")
|
||||
|
||||
logger.debug("Finished: test_api_proxmox_host_ipv4.")
|
||||
return False
|
||||
@@ -305,18 +308,16 @@ class ProxmoxApi:
|
||||
bool: True if the host is reachable on the specified port, False otherwise.
|
||||
"""
|
||||
logger.debug("Starting: test_api_proxmox_host_ipv6.")
|
||||
sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
|
||||
sock.settimeout(timeout)
|
||||
logger.warning(f"Host {host} ran into a timeout when connecting via IPv6 for tcp/{port}.")
|
||||
result = sock.connect_ex((host, port))
|
||||
|
||||
if result == 0:
|
||||
sock.close()
|
||||
ok, rc = Helper.tcp_connect_test(socket.AF_INET6, host, port, timeout)
|
||||
if ok:
|
||||
logger.debug(f"Host {host} is reachable on IPv6 for tcp/{port}.")
|
||||
logger.debug("Finished: test_api_proxmox_host_ipv6.")
|
||||
return True
|
||||
|
||||
sock.close()
|
||||
logger.warning(f"Host {host} is unreachable on IPv6 for tcp/{port}.")
|
||||
if rc == errno.ETIMEDOUT:
|
||||
logger.warning(f"Timeout connecting to {host} on IPv6 tcp/{port}.")
|
||||
else:
|
||||
logger.warning(f"Host {host} is unreachable on IPv6 for tcp/{port} (errno {rc}).")
|
||||
|
||||
logger.debug("Finished: test_api_proxmox_host_ipv6.")
|
||||
return False
|
||||
@@ -336,7 +337,15 @@ class ProxmoxApi:
|
||||
permissions_available = []
|
||||
|
||||
# Get the permissions for the current user/token from API
|
||||
permissions = proxmox_api.access.permissions.get()
|
||||
try:
|
||||
permissions = proxmox_api.access.permissions.get()
|
||||
except proxmoxer.core.ResourceException as api_error:
|
||||
if "no such user" in str(api_error):
|
||||
logger.error("Authentication to Proxmox API not possible: User not known - please check your username and config file.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
logger.error(f"Proxmox API error: {api_error}")
|
||||
sys.exit(1)
|
||||
|
||||
# Get all available permissions of the current user/token
|
||||
for path, permission in permissions.items():
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
__app_name__ = "ProxLB"
|
||||
__app_desc__ = "A DRS alike loadbalancer for Proxmox clusters."
|
||||
__app_desc__ = "An advanced resource scheduler and load balancer for Proxmox clusters."
|
||||
__author__ = "Florian Paul Azim Hoberg <gyptazy>"
|
||||
__copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
|
||||
__license__ = "GPL-3.0"
|
||||
__version__ = "1.1.5"
|
||||
__version__ = "1.1.11"
|
||||
__url__ = "https://github.com/gyptazy/ProxLB"
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
packaging
|
||||
proxmoxer
|
||||
requests
|
||||
urllib3
|
||||
PyYAML
|
||||
PyYAML
|
||||
|
||||
6
setup.py
6
setup.py
@@ -2,9 +2,9 @@ from setuptools import setup
|
||||
|
||||
setup(
|
||||
name="proxlb",
|
||||
version="1.1.5",
|
||||
description="A DRS alike loadbalancer for Proxmox clusters.",
|
||||
long_description="An advanced DRS alike loadbalancer for Proxmox clusters that also supports maintenance modes and affinity/anti-affinity rules.",
|
||||
version="1.1.11",
|
||||
description="An advanced resource scheduler and load balancer for Proxmox clusters.",
|
||||
long_description="An advanced resource scheduler and load balancer for Proxmox clusters that also supports maintenance modes and affinity/anti-affinity rules.",
|
||||
author="Florian Paul Azim Hoberg",
|
||||
author_email="gyptazy@gyptazy.com",
|
||||
maintainer="Florian Paul Azim Hoberg",
|
||||
|
||||
Reference in New Issue
Block a user