chore(E2E Tests): refactor verification scripts to use shared library functions and improve validation

2026-04-06 00:32:12 +02:00 · 2026-01-21 22:23:20 +00:00
parent 47bca3fb9b
commit 2d687a3275
6 changed files with 380 additions and 235 deletions
--- a/E2E/Terraform/e2e-tests/README.md
+++ b/E2E/Terraform/e2e-tests/README.md
@@ -10,15 +10,14 @@ e2e-tests/
 │   ├── index.sh              # Main entry point - orchestrates the full test flow
 │   ├── setup-test-account.sh # Creates test user, project, and API key
 │   ├── run-tests.sh          # Builds provider and runs all test cases
+│   ├── lib.sh                # Shared library with common test utilities
 │   └── cleanup.sh            # Cleans up test artifacts and state files
 └── tests/
    ├── 01-label/             # Label resource tests
    ├── 02-monitor-status/    # Monitor status resource tests
    ├── 03-incident-severity/ # Incident severity resource tests
-    ├── 04-incident-state/    # Incident state resource tests
-    ├── 05-status-page/       # Status page resource tests
-    ├── 06-alert-severity/    # Alert severity resource tests
-    └── 07-alert-state/       # Alert state resource tests
+    ├── ...                   # More test directories
+    └── XX-resource-name/     # Each test has main.tf, variables.tf, and verify.sh
 ```

 ## Running Tests
@@ -62,8 +61,67 @@ Each test case in `tests/` follows this pattern:
 1. `terraform init` - Initialize the Terraform configuration
 2. `terraform plan` - Create an execution plan
 3. `terraform apply` - Create the resources
-4. `terraform output` - Display created resource information
+4. `verify.sh` - Run API validation to verify resources were created correctly
 5. `terraform destroy` - Clean up created resources
+6. Verify deletion via API
+
+## Shared Library (lib.sh)
+
+The `scripts/lib.sh` file provides common utility functions for verify.sh scripts:
+
+### Helper Functions
+
+| Function | Description |
+|----------|-------------|
+| `unwrap_value "$json"` | Unwrap API values from wrapper format (e.g., `{"_type": "Color", "value": "#FF5733"}` → `#FF5733`) |
+| `get_output "name"` | Get a Terraform output value safely (returns empty string if not found) |
+| `assert_not_empty "$value" "name"` | Assert that a value is not empty |
+| `assert_equals "$expected" "$actual" "name"` | Assert two values are equal |
+| `api_get_resource "/api/endpoint" "$id" '{"select": true}'` | Make an API call to get a resource |
+| `verify_resource_exists "/api/endpoint" "$id"` | Verify a resource exists in the API |
+| `validate_field "$response" "field" "$expected"` | Validate a field from API response (handles wrapper unwrapping) |
+| `check_idempotency [strict]` | Run idempotency check (terraform plan should show no changes) |
+| `print_header "Test Name"` | Print test header |
+| `print_passed "Test Name"` | Print test passed message |
+| `print_failed "Test Name"` | Print test failed message and exit |
+
+### Example verify.sh
+
+```bash
+#!/bin/bash
+set -e
+
+# Source common library
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../../scripts/lib.sh"
+
+print_header "My Resource Verification"
+
+# Get terraform outputs
+RESOURCE_ID=$(get_output resource_id)
+EXPECTED_NAME=$(get_output resource_name)
+
+# Verify resource exists
+if ! verify_resource_exists "/api/my-resource" "$RESOURCE_ID"; then
+    print_failed "My Resource Verification"
+fi
+
+# Get full resource for validation
+RESPONSE=$(api_get_resource "/api/my-resource" "$RESOURCE_ID" '{"_id": true, "name": true}')
+
+# Validate fields
+validation_failed=0
+validate_field "$RESPONSE" "name" "$EXPECTED_NAME" || validation_failed=1
+
+if [ $validation_failed -eq 1 ]; then
+    print_failed "My Resource Verification"
+fi
+
+# Check idempotency
+check_idempotency true  # strict mode - fails on any changes
+
+print_passed "My Resource Verification"
+```

 ## Environment Variables

@@ -72,8 +130,8 @@ The following environment variables are used:
 | Variable | Default | Description |
 |----------|---------|-------------|
 | `ONEUPTIME_URL` | `http://localhost` | OneUptime instance URL |
-| `ONEUPTIME_API_KEY` | (generated) | API key for authentication |
-| `ONEUPTIME_PROJECT_ID` | (generated) | Project ID for resources |
+| `TF_VAR_api_key` | (generated) | API key for authentication |
+| `TF_VAR_project_id` | (generated) | Project ID for resources |

 ## CI/CD

@@ -89,4 +147,12 @@ See `.github/workflows/terraform-provider-e2e.yml` for the workflow configuratio
 1. Create a new directory in `tests/` with the naming convention `XX-resource-name/`
 2. Add `main.tf` with the Terraform configuration
 3. Add `variables.tf` with required variables
-4. Add the test directory name to the `TEST_DIRS` array in `scripts/run-tests.sh`
+4. Add `verify.sh` for API validation (source the shared library)
+5. Test directories are auto-discovered by `run-tests.sh`
+
+### Test Naming Convention
+
+- `01-XX` - Basic resource creation tests
+- `09-XX-crud` - Full CRUD operation tests
+- `XX-server-defaults` - Tests for server-computed default values
+- `XX-idempotency` - Tests for idempotency verification
--- a/E2E/Terraform/e2e-tests/scripts/lib.sh
+++ b/E2E/Terraform/e2e-tests/scripts/lib.sh
@@ -0,0 +1,185 @@
+#!/bin/bash
+# Common library functions for Terraform E2E tests
+# Source this file in verify.sh scripts: source "$(dirname "$0")/../../scripts/lib.sh"
+
+#######################################
+# Helper Functions
+#######################################
+
+# Unwrap API values that might be in wrapper format
+# e.g., {"_type": "Color", "value": "#FF5733"} -> "#FF5733"
+# Usage: unwrap_value "$raw_json_value"
+unwrap_value() {
+    local raw_value="$1"
+    if echo "$raw_value" | jq -e '.value' > /dev/null 2>&1; then
+        echo "$raw_value" | jq -r '.value'
+    else
+        echo "$raw_value" | jq -r '.'
+    fi
+}
+
+# Get a Terraform output value safely (returns empty string if not found)
+# Usage: get_output "output_name"
+get_output() {
+    local output_name="$1"
+    terraform output -raw "$output_name" 2>/dev/null || echo ""
+}
+
+# Assert that a value is not empty
+# Usage: assert_not_empty "$value" "Resource name"
+assert_not_empty() {
+    local value="$1"
+    local name="$2"
+    if [ -z "$value" ] || [ "$value" = "null" ]; then
+        echo "    ✗ FAILED: $name is empty or null"
+        return 1
+    fi
+    echo "    ✓ $name exists: $value"
+    return 0
+}
+
+# Assert two values are equal
+# Usage: assert_equals "$expected" "$actual" "Field name"
+assert_equals() {
+    local expected="$1"
+    local actual="$2"
+    local name="$3"
+    if [ "$expected" != "$actual" ]; then
+        echo "    ✗ FAILED: $name mismatch - Expected: '$expected', Got: '$actual'"
+        return 1
+    fi
+    echo "    ✓ $name matches: $actual"
+    return 0
+}
+
+# Make an API call to get a resource
+# Usage: api_get_resource "endpoint" "resource_id" "select_fields"
+# Example: api_get_resource "/api/label" "$LABEL_ID" '{"_id": true, "name": true}'
+api_get_resource() {
+    local endpoint="$1"
+    local resource_id="$2"
+    local select_fields="${3:-'{\"_id\": true}'}"
+
+    curl -s -X POST "${ONEUPTIME_URL}${endpoint}/${resource_id}/get-item" \
+        -H "Content-Type: application/json" \
+        -H "Apikey: $TF_VAR_api_key" \
+        -H "projectid: $TF_VAR_project_id" \
+        -d "{\"select\": $select_fields}"
+}
+
+# Verify a resource exists in the API
+# Usage: verify_resource_exists "endpoint" "resource_id"
+verify_resource_exists() {
+    local endpoint="$1"
+    local resource_id="$2"
+
+    local response
+    response=$(api_get_resource "$endpoint" "$resource_id" '{"_id": true}')
+
+    local api_id
+    api_id=$(echo "$response" | jq -r '._id // empty')
+
+    if [ -z "$api_id" ] || [ "$api_id" = "null" ]; then
+        echo "    ✗ FAILED: Resource not found in API"
+        echo "    Response: $response"
+        return 1
+    fi
+    echo "    ✓ Resource exists in API"
+    return 0
+}
+
+# Run idempotency check (terraform plan should show no changes)
+# Usage: check_idempotency [strict]
+# If strict=true, fails on any changes. Otherwise just warns.
+check_idempotency() {
+    local strict="${1:-false}"
+
+    echo ""
+    echo "  === Verifying idempotency ==="
+
+    local plan_output
+    local plan_exit_code
+    plan_output=$(terraform plan -detailed-exitcode 2>&1) || plan_exit_code=$?
+    plan_exit_code=${plan_exit_code:-0}
+
+    if [ "$plan_exit_code" -eq 0 ]; then
+        echo "  ✓ No changes detected - idempotency test PASSED"
+        return 0
+    elif [ "$plan_exit_code" -eq 2 ]; then
+        if [ "$strict" = "true" ]; then
+            echo "  ✗ FAILED: Changes detected after apply"
+            echo "  Plan output:"
+            echo "$plan_output"
+            return 1
+        else
+            echo "  ⚠ WARNING: Changes detected after apply"
+            echo "  This may indicate server defaults are being injected"
+            echo "  Plan output:"
+            echo "$plan_output"
+            return 0
+        fi
+    else
+        echo "  ✗ FAILED: terraform plan failed with exit code $plan_exit_code"
+        echo "$plan_output"
+        return 1
+    fi
+}
+
+# Validate a field from API response against expected value
+# Handles wrapper object unwrapping automatically
+# Usage: validate_field "$response" "field_name" "$expected_value"
+validate_field() {
+    local response="$1"
+    local field_name="$2"
+    local expected_value="$3"
+
+    local raw_value
+    raw_value=$(echo "$response" | jq ".$field_name")
+
+    local actual_value
+    actual_value=$(unwrap_value "$raw_value")
+
+    assert_equals "$expected_value" "$actual_value" "$field_name"
+}
+
+# Print test header
+# Usage: print_header "Test Name"
+print_header() {
+    local test_name="$1"
+    echo ""
+    echo "=========================================="
+    echo "$test_name"
+    echo "=========================================="
+}
+
+# Print test passed message
+# Usage: print_passed "Test Name"
+print_passed() {
+    local test_name="$1"
+    echo ""
+    echo "=== $test_name PASSED ==="
+}
+
+# Print test failed message and exit
+# Usage: print_failed "Test Name"
+print_failed() {
+    local test_name="$1"
+    echo ""
+    echo "=== $test_name FAILED ==="
+    exit 1
+}
+
+#######################################
+# Export functions for subshells
+#######################################
+export -f unwrap_value
+export -f get_output
+export -f assert_not_empty
+export -f assert_equals
+export -f api_get_resource
+export -f verify_resource_exists
+export -f check_idempotency
+export -f validate_field
+export -f print_header
+export -f print_passed
+export -f print_failed
--- a/E2E/Terraform/e2e-tests/tests/01-label/verify.sh
+++ b/E2E/Terraform/e2e-tests/tests/01-label/verify.sh
@@ -4,67 +4,38 @@

 set -e

-# Helper function to unwrap API values that might be in wrapper format
-# e.g., {"_type": "Color", "value": "#FF5733"} -> "#FF5733"
-unwrap_value() {
-    local raw_value="$1"
-    if echo "$raw_value" | jq -e '.value' > /dev/null 2>&1; then
-        echo "$raw_value" | jq -r '.value'
-    else
-        echo "$raw_value" | jq -r '.'
-    fi
-}
+# Source common library
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../../scripts/lib.sh"
+
+print_header "Label Resource Verification"

 # Get terraform outputs
-LABEL_ID=$(terraform output -raw label_id)
-LABEL_NAME=$(terraform output -raw label_name)
-LABEL_DESCRIPTION=$(terraform output -raw label_description)
-LABEL_COLOR=$(terraform output -raw label_color)
+LABEL_ID=$(get_output label_id)
+LABEL_NAME=$(get_output label_name)
+LABEL_DESCRIPTION=$(get_output label_description)
+LABEL_COLOR=$(get_output label_color)

 echo "  Verifying label resource via API..."
 echo "    Label ID: $LABEL_ID"

-# Call API to get the label
-RESPONSE=$(curl -s -X POST "${ONEUPTIME_URL}/api/label/${LABEL_ID}/get-item" \
-    -H "Content-Type: application/json" \
-    -H "Apikey: $TF_VAR_api_key" \
-    -H "projectid: $TF_VAR_project_id" \
-    -d '{"select": {"_id": true, "name": true, "description": true, "color": true}}')
-
-# Check if response contains the label
-API_ID=$(echo "$RESPONSE" | jq -r '._id // empty')
-if [ -z "$API_ID" ] || [ "$API_ID" = "null" ]; then
-    echo "    ✗ FAILED: Label not found in API response"
-    echo "    Response: $RESPONSE"
-    exit 1
+# Verify resource exists
+if ! verify_resource_exists "/api/label" "$LABEL_ID"; then
+    print_failed "Label Resource Verification"
 fi
-echo "    ✓ Label exists in API"

-# Validate name - handle wrapper object format
-API_NAME_RAW=$(echo "$RESPONSE" | jq '.name')
-API_NAME=$(unwrap_value "$API_NAME_RAW")
-if [ "$API_NAME" != "$LABEL_NAME" ]; then
-    echo "    ✗ FAILED: Name mismatch - Expected: '$LABEL_NAME', Got: '$API_NAME'"
-    exit 1
+# Get full resource for validation
+RESPONSE=$(api_get_resource "/api/label" "$LABEL_ID" '{"_id": true, "name": true, "description": true, "color": true}')
+
+# Validate all fields
+validation_failed=0
+
+validate_field "$RESPONSE" "name" "$LABEL_NAME" || validation_failed=1
+validate_field "$RESPONSE" "description" "$LABEL_DESCRIPTION" || validation_failed=1
+validate_field "$RESPONSE" "color" "$LABEL_COLOR" || validation_failed=1
+
+if [ $validation_failed -eq 1 ]; then
+    print_failed "Label Resource Verification"
 fi
-echo "    ✓ Name matches: $API_NAME"

-# Validate description - handle wrapper object format
-API_DESCRIPTION_RAW=$(echo "$RESPONSE" | jq '.description')
-API_DESCRIPTION=$(unwrap_value "$API_DESCRIPTION_RAW")
-if [ "$API_DESCRIPTION" != "$LABEL_DESCRIPTION" ]; then
-    echo "    ✗ FAILED: Description mismatch - Expected: '$LABEL_DESCRIPTION', Got: '$API_DESCRIPTION'"
-    exit 1
-fi
-echo "    ✓ Description matches: $API_DESCRIPTION"
-
-# Validate color - handle wrapper object format
-API_COLOR_RAW=$(echo "$RESPONSE" | jq '.color')
-API_COLOR=$(unwrap_value "$API_COLOR_RAW")
-if [ "$API_COLOR" != "$LABEL_COLOR" ]; then
-    echo "    ✗ FAILED: Color mismatch - Expected: '$LABEL_COLOR', Got: '$API_COLOR'"
-    exit 1
-fi
-echo "    ✓ Color matches: $API_COLOR"
-
-echo "    ✓ All label validations passed"
+print_passed "Label Resource Verification"
--- a/E2E/Terraform/e2e-tests/tests/09-label-crud/verify.sh
+++ b/E2E/Terraform/e2e-tests/tests/09-label-crud/verify.sh
@@ -4,66 +4,38 @@

 set -e

-# Helper function to unwrap API values that might be in wrapper format
-unwrap_value() {
-    local raw_value="$1"
-    if echo "$raw_value" | jq -e '.value' > /dev/null 2>&1; then
-        echo "$raw_value" | jq -r '.value'
-    else
-        echo "$raw_value" | jq -r '.'
-    fi
-}
+# Source common library
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../../scripts/lib.sh"
+
+print_header "Label CRUD Verification"

 # Get terraform outputs
-RESOURCE_ID=$(terraform output -raw label_id)
-EXPECTED_NAME=$(terraform output -raw label_name)
-EXPECTED_DESCRIPTION=$(terraform output -raw label_description)
-EXPECTED_COLOR=$(terraform output -raw label_color)
+RESOURCE_ID=$(get_output label_id)
+EXPECTED_NAME=$(get_output label_name)
+EXPECTED_DESCRIPTION=$(get_output label_description)
+EXPECTED_COLOR=$(get_output label_color)

 echo "  Verifying label CRUD resource via API..."
 echo "    Resource ID: $RESOURCE_ID"

-# Call API to get the resource
-RESPONSE=$(curl -s -X POST "${ONEUPTIME_URL}/api/label/${RESOURCE_ID}/get-item" \
-    -H "Content-Type: application/json" \
-    -H "Apikey: $TF_VAR_api_key" \
-    -H "projectid: $TF_VAR_project_id" \
-    -d '{"select": {"_id": true, "name": true, "description": true, "color": true}}')
-
-# Check if response contains the resource
-API_ID=$(echo "$RESPONSE" | jq -r '._id // empty')
-if [ -z "$API_ID" ] || [ "$API_ID" = "null" ]; then
-    echo "    ✗ FAILED: Label not found in API response"
-    echo "    Response: $RESPONSE"
-    exit 1
+# Verify resource exists
+if ! verify_resource_exists "/api/label" "$RESOURCE_ID"; then
+    print_failed "Label CRUD Verification"
 fi
-echo "    ✓ Label exists in API"

-# Validate name - handle wrapper object format
-API_NAME_RAW=$(echo "$RESPONSE" | jq '.name')
-API_NAME=$(unwrap_value "$API_NAME_RAW")
-if [ "$API_NAME" != "$EXPECTED_NAME" ]; then
-    echo "    ✗ FAILED: Name mismatch - Expected: '$EXPECTED_NAME', Got: '$API_NAME'"
-    exit 1
+# Get full resource for validation
+RESPONSE=$(api_get_resource "/api/label" "$RESOURCE_ID" '{"_id": true, "name": true, "description": true, "color": true}')
+
+# Validate all fields
+validation_failed=0
+
+validate_field "$RESPONSE" "name" "$EXPECTED_NAME" || validation_failed=1
+validate_field "$RESPONSE" "description" "$EXPECTED_DESCRIPTION" || validation_failed=1
+validate_field "$RESPONSE" "color" "$EXPECTED_COLOR" || validation_failed=1
+
+if [ $validation_failed -eq 1 ]; then
+    print_failed "Label CRUD Verification"
 fi
-echo "    ✓ Name matches: $API_NAME"

-# Validate description - handle wrapper object format
-API_DESCRIPTION_RAW=$(echo "$RESPONSE" | jq '.description')
-API_DESCRIPTION=$(unwrap_value "$API_DESCRIPTION_RAW")
-if [ "$API_DESCRIPTION" != "$EXPECTED_DESCRIPTION" ]; then
-    echo "    ✗ FAILED: Description mismatch - Expected: '$EXPECTED_DESCRIPTION', Got: '$API_DESCRIPTION'"
-    exit 1
-fi
-echo "    ✓ Description matches: $API_DESCRIPTION"
-
-# Validate color - handle wrapper object format
-API_COLOR_RAW=$(echo "$RESPONSE" | jq '.color')
-API_COLOR=$(unwrap_value "$API_COLOR_RAW")
-if [ "$API_COLOR" != "$EXPECTED_COLOR" ]; then
-    echo "    ✗ FAILED: Color mismatch - Expected: '$EXPECTED_COLOR', Got: '$API_COLOR'"
-    exit 1
-fi
-echo "    ✓ Color matches: $API_COLOR"
-
-echo "    ✓ All label CRUD validations passed"
+print_passed "Label CRUD Verification"
--- a/E2E/Terraform/e2e-tests/tests/22-monitor-crud/verify.sh
+++ b/E2E/Terraform/e2e-tests/tests/22-monitor-crud/verify.sh
@@ -1,74 +1,49 @@
 #!/bin/bash
+# Verify script for 22-monitor-crud test
+# Validates that monitor CRUD operations work correctly
+
 set -e

-# Test: Monitor CRUD Verification
-#
-# This script validates:
-# 1. All monitors were created successfully
-# 2. Monitors have expected attributes
-# 3. Server-computed fields are populated
+# Source common library
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../../scripts/lib.sh"

-echo "=== Monitor CRUD Test Verification ==="
+print_header "Monitor CRUD Verification"

 # Get terraform outputs
-MANUAL_BASIC_ID=$(terraform output -raw manual_basic_id 2>/dev/null || echo "")
-MANUAL_CUSTOM_ID=$(terraform output -raw manual_custom_id 2>/dev/null || echo "")
-WITH_LABELS_ID=$(terraform output -raw with_labels_id 2>/dev/null || echo "")
-LABEL_ID=$(terraform output -raw label_id 2>/dev/null || echo "")
-MONITOR_SLUG=$(terraform output -raw monitor_slug 2>/dev/null || echo "")
-MONITOR_STATUS_ID=$(terraform output -raw monitor_current_status_id 2>/dev/null || echo "")
+MANUAL_BASIC_ID=$(get_output manual_basic_id)
+MANUAL_CUSTOM_ID=$(get_output manual_custom_id)
+WITH_LABELS_ID=$(get_output with_labels_id)
+LABEL_ID=$(get_output label_id)
+MONITOR_SLUG=$(get_output monitor_slug)
+MONITOR_STATUS_ID=$(get_output monitor_current_status_id)

-echo "Manual Basic Monitor ID: $MANUAL_BASIC_ID"
-echo "Manual Custom Monitor ID: $MANUAL_CUSTOM_ID"
-echo "Monitor with Labels ID: $WITH_LABELS_ID"
-echo "Label ID: $LABEL_ID"
-echo "Monitor Slug: $MONITOR_SLUG"
-echo "Current Monitor Status ID: $MONITOR_STATUS_ID"
+echo "  Monitor IDs:"
+echo "    Manual Basic: $MANUAL_BASIC_ID"
+echo "    Manual Custom: $MANUAL_CUSTOM_ID"
+echo "    With Labels: $WITH_LABELS_ID"
+echo "    Label ID: $LABEL_ID"
+echo "    Slug: $MONITOR_SLUG"
+echo "    Status ID: $MONITOR_STATUS_ID"

 # Verify all monitors were created
-if [ -z "$MANUAL_BASIC_ID" ]; then
-    echo "ERROR: manual_basic monitor was not created"
-    exit 1
-fi
+validation_failed=0

-if [ -z "$MANUAL_CUSTOM_ID" ]; then
-    echo "ERROR: manual_custom monitor was not created"
-    exit 1
-fi
+assert_not_empty "$MANUAL_BASIC_ID" "Manual Basic Monitor" || validation_failed=1
+assert_not_empty "$MANUAL_CUSTOM_ID" "Manual Custom Monitor" || validation_failed=1
+assert_not_empty "$WITH_LABELS_ID" "Monitor with Labels" || validation_failed=1
+assert_not_empty "$LABEL_ID" "Label" || validation_failed=1

-if [ -z "$WITH_LABELS_ID" ]; then
-    echo "ERROR: monitor with labels was not created"
-    exit 1
-fi
-
-if [ -z "$LABEL_ID" ]; then
-    echo "ERROR: label was not created"
-    exit 1
-fi
-
-# Verify server-computed fields are populated
+# Server-computed fields (warning only)
 if [ -z "$MONITOR_SLUG" ]; then
-    echo "WARNING: Monitor slug is empty - server may not have generated it"
+    echo "    ⚠ WARNING: Monitor slug is empty - server may not have generated it"
 fi

-echo ""
-echo "=== Verifying idempotency ==="
-PLAN_OUTPUT=$(terraform plan -detailed-exitcode 2>&1) || PLAN_EXIT_CODE=$?
-PLAN_EXIT_CODE=${PLAN_EXIT_CODE:-0}
-
-if [ "$PLAN_EXIT_CODE" -eq 0 ]; then
-    echo "SUCCESS: No changes detected - idempotency test PASSED"
-elif [ "$PLAN_EXIT_CODE" -eq 2 ]; then
-    echo "WARNING: Changes detected after apply"
-    echo "This may indicate server defaults are being injected"
-    echo "Plan output:"
-    echo "$PLAN_OUTPUT"
-    # Don't fail for now as some fields may have server defaults
-else
-    echo "ERROR: terraform plan failed with exit code $PLAN_EXIT_CODE"
-    echo "$PLAN_OUTPUT"
-    exit 1
+if [ $validation_failed -eq 1 ]; then
+    print_failed "Monitor CRUD Verification"
 fi

-echo ""
-echo "=== Monitor CRUD Test PASSED ==="
+# Check idempotency (non-strict mode - some server defaults may cause changes)
+check_idempotency false
+
+print_passed "Monitor CRUD Verification"
--- a/E2E/Terraform/e2e-tests/tests/28-incident-crud/verify.sh
+++ b/E2E/Terraform/e2e-tests/tests/28-incident-crud/verify.sh
@@ -1,77 +1,53 @@
 #!/bin/bash
+# Verify script for 28-incident-crud test
+# Validates that incident CRUD operations work correctly
+
 set -e

-echo "=== Incident CRUD Test Verification ==="
+# Source common library
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../../scripts/lib.sh"
+
+print_header "Incident CRUD Verification"

 # Get outputs
-BASIC_ID=$(terraform output -raw basic_incident_id 2>/dev/null || echo "")
-ROOT_CAUSE_ID=$(terraform output -raw with_root_cause_id 2>/dev/null || echo "")
-VISIBILITY_ID=$(terraform output -raw visibility_settings_id 2>/dev/null || echo "")
-LABELED_ID=$(terraform output -raw with_labels_id 2>/dev/null || echo "")
-SEVERITY_ID=$(terraform output -raw severity_id 2>/dev/null || echo "")
-STATE_ID=$(terraform output -raw state_id 2>/dev/null || echo "")
-ROOT_CAUSE=$(terraform output -raw with_root_cause_root_cause 2>/dev/null || echo "")
+BASIC_ID=$(get_output basic_incident_id)
+ROOT_CAUSE_ID=$(get_output with_root_cause_id)
+VISIBILITY_ID=$(get_output visibility_settings_id)
+LABELED_ID=$(get_output with_labels_id)
+SEVERITY_ID=$(get_output severity_id)
+STATE_ID=$(get_output state_id)
+ROOT_CAUSE=$(get_output with_root_cause_root_cause)

-echo "Basic Incident ID: $BASIC_ID"
-echo "Root Cause Incident ID: $ROOT_CAUSE_ID"
-echo "Visibility Incident ID: $VISIBILITY_ID"
-echo "Labeled Incident ID: $LABELED_ID"
-echo "Severity ID: $SEVERITY_ID"
-echo "State ID: $STATE_ID"
-echo "Root Cause: $ROOT_CAUSE"
+echo "  Incident IDs:"
+echo "    Basic: $BASIC_ID"
+echo "    Root Cause: $ROOT_CAUSE_ID"
+echo "    Visibility: $VISIBILITY_ID"
+echo "    Labeled: $LABELED_ID"
+echo "    Severity ID: $SEVERITY_ID"
+echo "    State ID: $STATE_ID"
+echo "    Root Cause Value: $ROOT_CAUSE"

 # Verify all resources created
-if [ -z "$BASIC_ID" ]; then
-    echo "ERROR: Basic incident not created"
-    exit 1
-fi
+validation_failed=0

-if [ -z "$ROOT_CAUSE_ID" ]; then
-    echo "ERROR: Root cause incident not created"
-    exit 1
-fi
+assert_not_empty "$BASIC_ID" "Basic Incident" || validation_failed=1
+assert_not_empty "$ROOT_CAUSE_ID" "Root Cause Incident" || validation_failed=1
+assert_not_empty "$VISIBILITY_ID" "Visibility Incident" || validation_failed=1
+assert_not_empty "$LABELED_ID" "Labeled Incident" || validation_failed=1
+assert_not_empty "$SEVERITY_ID" "Severity" || validation_failed=1
+assert_not_empty "$STATE_ID" "State" || validation_failed=1

-if [ -z "$VISIBILITY_ID" ]; then
-    echo "ERROR: Visibility incident not created"
-    exit 1
-fi
-
-if [ -z "$LABELED_ID" ]; then
-    echo "ERROR: Labeled incident not created"
-    exit 1
-fi
-
-if [ -z "$SEVERITY_ID" ]; then
-    echo "ERROR: Severity not created"
-    exit 1
-fi
-
-if [ -z "$STATE_ID" ]; then
-    echo "ERROR: State not created"
-    exit 1
-fi
-
-# Verify root cause is set correctly
+# Root cause (warning only)
 if [ -z "$ROOT_CAUSE" ]; then
-    echo "WARNING: Root cause is empty"
+    echo "    ⚠ WARNING: Root cause is empty"
 fi

-echo ""
-echo "=== Verifying idempotency ==="
-PLAN_OUTPUT=$(terraform plan -detailed-exitcode 2>&1) || PLAN_EXIT_CODE=$?
-PLAN_EXIT_CODE=${PLAN_EXIT_CODE:-0}
-
-if [ "$PLAN_EXIT_CODE" -eq 0 ]; then
-    echo "SUCCESS: No changes detected - idempotency test PASSED"
-elif [ "$PLAN_EXIT_CODE" -eq 2 ]; then
-    echo "ERROR: Changes detected after apply"
-    echo "Plan output:"
-    echo "$PLAN_OUTPUT"
-    exit 1
-else
-    echo "ERROR: terraform plan failed"
-    exit 1
+if [ $validation_failed -eq 1 ]; then
+    print_failed "Incident CRUD Verification"
 fi

-echo ""
-echo "=== Incident CRUD Test PASSED ==="
+# Check idempotency (strict mode - incident should be fully idempotent)
+check_idempotency true
+
+print_passed "Incident CRUD Verification"