fix performance regression for Site save, use bulk_update for cached fields #578

New Issue

MrUnknownDE · 2026-04-05T16:46:46+02:00

MrUnknownDE commented

2026-04-05 16:46:46 +02:00

Originally created by @arthanson on 1/15/2026

Fixes: #21118

Use bulk_update to update cached fields prevent N+1 queries. Here is an AI generated management command I used to test the issue. Just run with defaults and old code takes several minutes when you update the Site, new code does it in several seconds.

from django.core.management.base import BaseCommand
from django.db import transaction

from dcim.models import Device, DeviceRole, DeviceType, Location, Manufacturer, Site
from ipam.models import Prefix, VLAN, VLANGroup


class Command(BaseCommand):
    help = """
    Generate a test site with a large number of related objects for performance testing.

    This creates a site with many prefixes to reproduce issue #21118, where updating a site
    with thousands of related Prefix/Cluster/WirelessLAN objects causes timeouts due to
    O(N) performance in the sync_cached_scope_fields signal handler.
    """

    def add_arguments(self, parser):
        parser.add_argument(
            '--devices',
            type=int,
            default=1250,
            help='Number of devices to create (default: 1250)'
        )
        parser.add_argument(
            '--prefixes',
            type=int,
            default=4000,
            help='Number of prefixes to create (default: 4000)'
        )
        parser.add_argument(
            '--vlans',
            type=int,
            default=20,
            help='Number of VLANs to create (default: 20)'
        )
        parser.add_argument(
            '--name',
            type=str,
            default='Test Site Large',
            help='Site name (default: "Test Site Large")'
        )

    def handle(self, *args, **options):
        devices_count = options['devices']
        prefixes_count = options['prefixes']
        vlans_count = options['vlans']
        site_name = options['name']
        site_slug = site_name.lower().replace(' ', '-')

        self.stdout.write(f"Creating site '{site_name}' with:")
        self.stdout.write(f"  - {devices_count} devices")
        self.stdout.write(f"  - {prefixes_count} prefixes")
        self.stdout.write(f"  - 1 VLAN group")
        self.stdout.write(f"  - {vlans_count} VLANs")
        self.stdout.write("")

        try:
            with transaction.atomic():
                # Create site
                self.stdout.write("Creating site...")
                site = Site.objects.create(
                    name=site_name,
                    slug=site_slug,
                    description=f"Test site with {prefixes_count} prefixes for performance testing"
                )

                # Get or create a single location for device assignment
                location = Location.objects.filter(site=site).first()
                if not location:
                    self.stdout.write("Creating location for devices...")
                    location = Location.objects.create(
                        name=f'{site_name} Location',
                        slug=f'{site_slug}-location',
                        site=site
                    )
                    self.stdout.write(self.style.SUCCESS("  ✓ Created location"))
                else:
                    self.stdout.write(f"Using existing location: {location.name}")

                # Get or create manufacturer, device type, and device role (needed for devices)
                self.stdout.write("Getting/creating device prerequisites...")
                manufacturer = Manufacturer.objects.first()
                if not manufacturer:
                    manufacturer = Manufacturer.objects.create(
                        name='Test Manufacturer',
                        slug='test-manufacturer'
                    )

                device_type = DeviceType.objects.filter(manufacturer=manufacturer).first()
                if not device_type:
                    device_type = DeviceType.objects.create(
                        manufacturer=manufacturer,
                        model='Test Switch',
                        slug='test-switch'
                    )

                device_role = DeviceRole.objects.first()
                if not device_role:
                    device_role = DeviceRole.objects.create(
                        name='Test Role',
                        slug='test-role'
                    )

                # Create devices
                self.stdout.write(f"Creating {devices_count} devices...")
                devices = []
                for i in range(devices_count):
                    devices.append(
                        Device(
                            name=f'{site_slug}-device-{i+1:04d}',
                            site=site,
                            location=location,
                            device_type=device_type,
                            role=device_role
                        )
                    )
                    if len(devices) >= 500:
                        Device.objects.bulk_create(devices)
                        self.stdout.write(f"  Created {len(devices)} devices...")
                        devices = []
                if devices:
                    Device.objects.bulk_create(devices)
                self.stdout.write(self.style.SUCCESS(f"  ✓ Created {devices_count} devices"))

                # Create prefixes
                self.stdout.write(f"Creating {prefixes_count} prefixes...")
                from django.contrib.contenttypes.models import ContentType
                site_ct = ContentType.objects.get_for_model(Site)

                prefixes = []
                for i in range(prefixes_count):
                    # Create diverse prefixes using 10.0.0.0/8 space
                    # Each /24 gives us 256 IPs, we can fit 65536 /24s in 10.0.0.0/8
                    octet2 = i // 256
                    octet3 = i % 256
                    prefix = Prefix(
                        prefix=f'10.{octet2}.{octet3}.0/24',
                        scope_type=site_ct,
                        scope_id=site.pk,
                        # Manually populate cached fields since bulk_create bypasses save()
                        _site=site,
                        _region=site.region,
                        _site_group=site.group
                    )
                    prefixes.append(prefix)
                    if len(prefixes) >= 500:
                        Prefix.objects.bulk_create(prefixes)
                        self.stdout.write(f"  Created {len(prefixes)} prefixes...")
                        prefixes = []
                if prefixes:
                    Prefix.objects.bulk_create(prefixes)
                self.stdout.write(self.style.SUCCESS(f"  ✓ Created {prefixes_count} prefixes"))

                # Create VLAN group
                self.stdout.write("Creating VLAN group...")
                vlan_group = VLANGroup.objects.create(
                    name=f'{site_name} VLANs',
                    slug=f'{site_slug}-vlans',
                    scope=site
                )
                self.stdout.write(self.style.SUCCESS("  ✓ Created VLAN group"))

                # Create VLANs
                self.stdout.write(f"Creating {vlans_count} VLANs...")
                vlans = []
                for i in range(vlans_count):
                    vlans.append(
                        VLAN(
                            vid=i + 100,  # Start from VLAN 100
                            name=f'VLAN-{i + 100}',
                            group=vlan_group,
                            site=site
                        )
                    )
                VLAN.objects.bulk_create(vlans)
                self.stdout.write(self.style.SUCCESS(f"  ✓ Created {vlans_count} VLANs"))

                self.stdout.write("")
                self.stdout.write(self.style.SUCCESS("=" * 60))
                self.stdout.write(self.style.SUCCESS(f"Successfully created site!"))
                self.stdout.write(self.style.SUCCESS(f"Site ID: {site.pk}"))
                self.stdout.write(self.style.SUCCESS(f"Site Name: {site.name}"))
                self.stdout.write(self.style.SUCCESS("=" * 60))

        except Exception as e:
            self.stdout.write(self.style.ERROR(f"Error creating site: {e}"))
            raise

*Originally created by @arthanson on 1/15/2026* ### Fixes: #21118 Use bulk_update to update cached fields prevent N+1 queries. Here is an AI generated management command I used to test the issue. Just run with defaults and old code takes several minutes when you update the Site, new code does it in several seconds. ``` from django.core.management.base import BaseCommand from django.db import transaction from dcim.models import Device, DeviceRole, DeviceType, Location, Manufacturer, Site from ipam.models import Prefix, VLAN, VLANGroup class Command(BaseCommand): help = """ Generate a test site with a large number of related objects for performance testing. This creates a site with many prefixes to reproduce issue #21118, where updating a site with thousands of related Prefix/Cluster/WirelessLAN objects causes timeouts due to O(N) performance in the sync_cached_scope_fields signal handler. """ def add_arguments(self, parser): parser.add_argument( '--devices', type=int, default=1250, help='Number of devices to create (default: 1250)' ) parser.add_argument( '--prefixes', type=int, default=4000, help='Number of prefixes to create (default: 4000)' ) parser.add_argument( '--vlans', type=int, default=20, help='Number of VLANs to create (default: 20)' ) parser.add_argument( '--name', type=str, default='Test Site Large', help='Site name (default: "Test Site Large")' ) def handle(self, *args, **options): devices_count = options['devices'] prefixes_count = options['prefixes'] vlans_count = options['vlans'] site_name = options['name'] site_slug = site_name.lower().replace(' ', '-') self.stdout.write(f"Creating site '{site_name}' with:") self.stdout.write(f" - {devices_count} devices") self.stdout.write(f" - {prefixes_count} prefixes") self.stdout.write(f" - 1 VLAN group") self.stdout.write(f" - {vlans_count} VLANs") self.stdout.write("") try: with transaction.atomic(): # Create site self.stdout.write("Creating site...") site = Site.objects.create( name=site_name, slug=site_slug, description=f"Test site with {prefixes_count} prefixes for performance testing" ) # Get or create a single location for device assignment location = Location.objects.filter(site=site).first() if not location: self.stdout.write("Creating location for devices...") location = Location.objects.create( name=f'{site_name} Location', slug=f'{site_slug}-location', site=site ) self.stdout.write(self.style.SUCCESS(" ✓ Created location")) else: self.stdout.write(f"Using existing location: {location.name}") # Get or create manufacturer, device type, and device role (needed for devices) self.stdout.write("Getting/creating device prerequisites...") manufacturer = Manufacturer.objects.first() if not manufacturer: manufacturer = Manufacturer.objects.create( name='Test Manufacturer', slug='test-manufacturer' ) device_type = DeviceType.objects.filter(manufacturer=manufacturer).first() if not device_type: device_type = DeviceType.objects.create( manufacturer=manufacturer, model='Test Switch', slug='test-switch' ) device_role = DeviceRole.objects.first() if not device_role: device_role = DeviceRole.objects.create( name='Test Role', slug='test-role' ) # Create devices self.stdout.write(f"Creating {devices_count} devices...") devices = [] for i in range(devices_count): devices.append( Device( name=f'{site_slug}-device-{i+1:04d}', site=site, location=location, device_type=device_type, role=device_role ) ) if len(devices) >= 500: Device.objects.bulk_create(devices) self.stdout.write(f" Created {len(devices)} devices...") devices = [] if devices: Device.objects.bulk_create(devices) self.stdout.write(self.style.SUCCESS(f" ✓ Created {devices_count} devices")) # Create prefixes self.stdout.write(f"Creating {prefixes_count} prefixes...") from django.contrib.contenttypes.models import ContentType site_ct = ContentType.objects.get_for_model(Site) prefixes = [] for i in range(prefixes_count): # Create diverse prefixes using 10.0.0.0/8 space # Each /24 gives us 256 IPs, we can fit 65536 /24s in 10.0.0.0/8 octet2 = i // 256 octet3 = i % 256 prefix = Prefix( prefix=f'10.{octet2}.{octet3}.0/24', scope_type=site_ct, scope_id=site.pk, # Manually populate cached fields since bulk_create bypasses save() _site=site, _region=site.region, _site_group=site.group ) prefixes.append(prefix) if len(prefixes) >= 500: Prefix.objects.bulk_create(prefixes) self.stdout.write(f" Created {len(prefixes)} prefixes...") prefixes = [] if prefixes: Prefix.objects.bulk_create(prefixes) self.stdout.write(self.style.SUCCESS(f" ✓ Created {prefixes_count} prefixes")) # Create VLAN group self.stdout.write("Creating VLAN group...") vlan_group = VLANGroup.objects.create( name=f'{site_name} VLANs', slug=f'{site_slug}-vlans', scope=site ) self.stdout.write(self.style.SUCCESS(" ✓ Created VLAN group")) # Create VLANs self.stdout.write(f"Creating {vlans_count} VLANs...") vlans = [] for i in range(vlans_count): vlans.append( VLAN( vid=i + 100, # Start from VLAN 100 name=f'VLAN-{i + 100}', group=vlan_group, site=site ) ) VLAN.objects.bulk_create(vlans) self.stdout.write(self.style.SUCCESS(f" ✓ Created {vlans_count} VLANs")) self.stdout.write("") self.stdout.write(self.style.SUCCESS("=" * 60)) self.stdout.write(self.style.SUCCESS(f"Successfully created site!")) self.stdout.write(self.style.SUCCESS(f"Site ID: {site.pk}")) self.stdout.write(self.style.SUCCESS(f"Site Name: {site.name}")) self.stdout.write(self.style.SUCCESS("=" * 60)) except Exception as e: self.stdout.write(self.style.ERROR(f"Error creating site: {e}")) raise ```

MrUnknownDE closed this issue

2026-04-05 16:46:46 +02:00

Sign in to join this conversation.

Branches Tags

main

21455-sql-indexes-audit

21357-register-model-actions

feature

20924-plugin-ui-components

21766-improve-test-coverage-for-sortable-table-columns

21157-export-template-public-models

21025-pre-render-config-contexts

21364-swagger

feature-ip-prefix-link

20911-dropdown-3

fix_module_substitution

21160-filterset

20911-dropdown-2

20044-elevation-stuck-lightmode

7604-filter-modifiers-v3

20660-script-load

19724-graphql

14884-script

19720-macaddress-interface-generic-relation

fix-19669-api-image-download

7604-filter-modifiers

19275-fixes-interface-bulk-edit

fix-17794-get_field_value_return_list

11507-show-aggregate-and-rir-on-api

9583-add_column_specific_search_field_to_tables

No Label

1 Participants

Notifications

Due Date

No due date set.

Dependencies

No dependencies set.

Reference: github/netbox#578