#21160 fix perf issue for filterset rendering of APISelect widgets #546

New Issue

MrUnknownDE · 2026-04-05T16:42:34+02:00

MrUnknownDE commented

2026-04-05 16:42:34 +02:00

Originally created by @arthanson on 1/16/2026

Fixes: #21160

In FilterModifierWidget, the get_context was causing the original Widget to render with the entire queryset server side which is a problem for APISelect, APISelectMultiple as those can be huge querysets and are rendered dynamically on the front-end calling the API. On this render/init we just need to get the items that are currently selected, the select widget will call the API as normally when it is dropped down.

Here is an AI generated management command I used to load up interfaces. Run this (I used 500000 interfaces) and go to the interfaces list view to see the problem:

import time
from django.core.management.base import BaseCommand
from django.db import transaction

from dcim.choices import InterfaceTypeChoices
from dcim.models import Site, Manufacturer, DeviceType, DeviceRole, Device, Interface


class Command(BaseCommand):
    help = """
    Generate test data to reproduce interface page performance issues (GitHub #21160).

    This command creates a large number of interfaces with parent/bridge/LAG relationships
    to demonstrate the performance regression caused by unnecessary FilterSet instantiation.
    """

    def add_arguments(self, parser):
        parser.add_argument(
            '--count',
            type=int,
            default=100000,
            help='Number of interfaces to create (default: 100000)'
        )
        parser.add_argument(
            '--no-input',
            action='store_true',
            help='Do not prompt for confirmation'
        )

    def draw_progress_bar(self, current, total, label='Progress'):
        """Draw a simple progress bar."""
        percentage = (current * 100) / total
        bar_size = int(percentage / 5)
        self.stdout.write(
            f"\r  {label}: [{'#' * bar_size}{' ' * (20 - bar_size)}] {int(percentage)}% ({current}/{total})",
            ending=''
        )

    def create_test_data(self, count, options):
        """Create test interfaces and related data."""
        self.stdout.write(self.style.SUCCESS(f"Creating test data for {count} interfaces..."))

        # Create prerequisite objects
        self.stdout.write("Creating prerequisite objects...")

        site, _ = Site.objects.get_or_create(
            name='Test Performance Site 1',
            defaults={'slug': 'test-perf-site-1'}
        )

        manufacturer, _ = Manufacturer.objects.get_or_create(
            name='Test Performance Manufacturer',
            defaults={'slug': 'test-perf-mfr'}
        )

        device_type, _ = DeviceType.objects.get_or_create(
            manufacturer=manufacturer,
            model='Test Performance Switch',
            defaults={'slug': 'test-perf-switch'}
        )

        role, _ = DeviceRole.objects.get_or_create(
            name='Test Performance Role',
            defaults={'slug': 'test-perf-role'}
        )

        # Calculate how many devices we need (48 interfaces per device)
        interfaces_per_device = 48
        num_devices = (count + interfaces_per_device - 1) // interfaces_per_device

        self.stdout.write(f"Creating {num_devices} devices...")

        # Create devices
        devices = []
        for i in range(num_devices):
            devices.append(Device(
                name=f'test-perf-device-{i:05d}',
                device_type=device_type,
                role=role,
                site=site
            ))

        # Bulk create devices
        start_time = time.time()
        with transaction.atomic():
            Device.objects.bulk_create(devices, batch_size=1000)
        device_creation_time = time.time() - start_time
        self.stdout.write(self.style.SUCCESS(f"  Created {num_devices} devices in {device_creation_time:.2f}s"))

        # Fetch created devices back (we need their PKs)
        devices = list(Device.objects.filter(name__startswith='test-perf-device-').order_by('name'))

        # Create interfaces
        self.stdout.write(f"Creating {count} interfaces...")
        interfaces = []
        interface_types = [
            InterfaceTypeChoices.TYPE_1GE_FIXED,
            InterfaceTypeChoices.TYPE_10GE_FIXED,
            InterfaceTypeChoices.TYPE_25GE_SR,
            InterfaceTypeChoices.TYPE_40GE_SR4,
            InterfaceTypeChoices.TYPE_100GE_SR4,
        ]

        created_count = 0
        device_idx = 0
        port_num = 1

        start_time = time.time()

        for i in range(count):
            if port_num > interfaces_per_device:
                device_idx += 1
                port_num = 1
                if device_idx >= len(devices):
                    break

            interface = Interface(
                device=devices[device_idx],
                name=f'eth{port_num}',
                type=interface_types[i % len(interface_types)],
                enabled=True
            )
            interfaces.append(interface)
            port_num += 1
            created_count += 1

            # Show progress every 1000 interfaces
            if created_count % 1000 == 0:
                self.draw_progress_bar(created_count, count, 'Creating interfaces')

        self.draw_progress_bar(created_count, count, 'Creating interfaces')
        self.stdout.write()  # newline

        # Bulk create interfaces in batches
        with transaction.atomic():
            Interface.objects.bulk_create(interfaces, batch_size=1000)

        interface_creation_time = time.time() - start_time
        self.stdout.write(self.style.SUCCESS(f"  Created {created_count} interfaces in {interface_creation_time:.2f}s"))

        # Fetch created interfaces to set up relationships
        all_interfaces = list(Interface.objects.filter(device__in=devices).order_by('id'))

        if len(all_interfaces) < 100:
            self.stdout.write(self.style.WARNING("  Not enough interfaces to create relationships"))
            return

        # Create LAG interfaces and assign members
        self.stdout.write("Creating LAG relationships...")
        num_lags = min(100, len(all_interfaces) // 10)
        lag_updates = []

        for i in range(num_lags):
            # Every 10th interface becomes a LAG
            lag_interface = all_interfaces[i * 10]
            lag_interface.type = InterfaceTypeChoices.TYPE_LAG
            lag_updates.append(lag_interface)

            # Next 3 interfaces become LAG members
            for j in range(1, 4):
                idx = i * 10 + j
                if idx < len(all_interfaces):
                    member = all_interfaces[idx]
                    member.lag = lag_interface
                    lag_updates.append(member)

        with transaction.atomic():
            Interface.objects.bulk_update(lag_updates, ['type', 'lag'], batch_size=500)

        self.stdout.write(self.style.SUCCESS(f"  Created {num_lags} LAG interfaces with members"))

        # Create bridge relationships
        self.stdout.write("Creating bridge relationships...")
        bridge_updates = []
        num_bridges = min(50, len(all_interfaces) // 20)

        for i in range(num_bridges):
            # Create bridge interfaces
            bridge_idx = i * 20 + 5
            if bridge_idx >= len(all_interfaces):
                break

            bridge_interface = all_interfaces[bridge_idx]
            bridge_interface.type = InterfaceTypeChoices.TYPE_BRIDGE
            bridge_updates.append(bridge_interface)

            # Bridge 2 interfaces to it
            for j in range(1, 3):
                member_idx = bridge_idx + j
                if member_idx < len(all_interfaces):
                    member = all_interfaces[member_idx]
                    member.bridge = bridge_interface
                    bridge_updates.append(member)

        with transaction.atomic():
            Interface.objects.bulk_update(bridge_updates, ['type', 'bridge'], batch_size=500)

        self.stdout.write(self.style.SUCCESS(f"  Created {num_bridges} bridge interfaces"))

        # Create parent-child relationships
        self.stdout.write("Creating parent-child relationships (subinterfaces)...")
        parent_updates = []
        num_parents = min(100, len(all_interfaces) // 15)

        for i in range(num_parents):
            parent_idx = i * 15
            if parent_idx >= len(all_interfaces):
                break

            parent = all_interfaces[parent_idx]

            # Create 2-3 subinterfaces
            for j in range(1, 4):
                child_idx = parent_idx + j + 10
                if child_idx < len(all_interfaces):
                    child = all_interfaces[child_idx]
                    child.parent = parent
                    # Change name to subinterface format
                    child.name = f'{parent.name}.{j}'
                    parent_updates.append(child)

        with transaction.atomic():
            Interface.objects.bulk_update(parent_updates, ['parent', 'name'], batch_size=500)

        self.stdout.write(
            self.style.SUCCESS(f"  Created parent-child relationships for {len(parent_updates)} interfaces")
        )

        total_time = device_creation_time + interface_creation_time
        self.stdout.write()
        self.stdout.write(self.style.SUCCESS("=" * 70))
        self.stdout.write(self.style.SUCCESS("Test data creation complete!"))
        self.stdout.write(self.style.SUCCESS(f"Total time: {total_time:.2f}s"))
        self.stdout.write()
        self.stdout.write("Created:")
        self.stdout.write(f"  - {created_count} total interfaces")
        self.stdout.write(f"  - {num_lags} LAG interfaces with members")
        self.stdout.write(f"  - {num_bridges} bridge interfaces")
        self.stdout.write(f"  - {len(parent_updates)} subinterfaces with parents")
        self.stdout.write()
        self.stdout.write("Now visit /dcim/interfaces/ in your browser to test page load time.")
        self.stdout.write()
        self.stdout.write(f"With {created_count:,} interfaces, the performance issue should be visible.")
        self.stdout.write("With the fix applied, the page should load normally.")
        self.stdout.write("Without the fix (v4.5.0), it would timeout or take a long time to load.")
        self.stdout.write("=" * 70)

    def handle(self, *args, **options):
        count = options['count']
        if count < 50000:
            self.stdout.write(self.style.WARNING(
                f"Warning: {count} interfaces may not show significant performance difference."
            ))
            self.stdout.write("The issue becomes apparent with 100k+ interfaces.")
            self.stdout.write("Consider using --count=100000 or higher for best demonstration.")
            if not options['no_input']:
                confirmation = input("Continue anyway? (yes/no): ")
                if confirmation != 'yes':
                    self.stdout.write(self.style.SUCCESS("Aborting."))
                    return

        self.create_test_data(count, options)

*Originally created by @arthanson on 1/16/2026* ### Fixes: #21160 In FilterModifierWidget, the get_context was causing the original Widget to render with the entire queryset server side which is a problem for APISelect, APISelectMultiple as those can be huge querysets and are rendered dynamically on the front-end calling the API. On this render/init we just need to get the items that are currently selected, the select widget will call the API as normally when it is dropped down. Here is an AI generated management command I used to load up interfaces. Run this (I used 500000 interfaces) and go to the interfaces list view to see the problem: ``` import time from django.core.management.base import BaseCommand from django.db import transaction from dcim.choices import InterfaceTypeChoices from dcim.models import Site, Manufacturer, DeviceType, DeviceRole, Device, Interface class Command(BaseCommand): help = """ Generate test data to reproduce interface page performance issues (GitHub #21160). This command creates a large number of interfaces with parent/bridge/LAG relationships to demonstrate the performance regression caused by unnecessary FilterSet instantiation. """ def add_arguments(self, parser): parser.add_argument( '--count', type=int, default=100000, help='Number of interfaces to create (default: 100000)' ) parser.add_argument( '--no-input', action='store_true', help='Do not prompt for confirmation' ) def draw_progress_bar(self, current, total, label='Progress'): """Draw a simple progress bar.""" percentage = (current * 100) / total bar_size = int(percentage / 5) self.stdout.write( f"\r {label}: [{'#' * bar_size}{' ' * (20 - bar_size)}] {int(percentage)}% ({current}/{total})", ending='' ) def create_test_data(self, count, options): """Create test interfaces and related data.""" self.stdout.write(self.style.SUCCESS(f"Creating test data for {count} interfaces...")) # Create prerequisite objects self.stdout.write("Creating prerequisite objects...") site, _ = Site.objects.get_or_create( name='Test Performance Site 1', defaults={'slug': 'test-perf-site-1'} ) manufacturer, _ = Manufacturer.objects.get_or_create( name='Test Performance Manufacturer', defaults={'slug': 'test-perf-mfr'} ) device_type, _ = DeviceType.objects.get_or_create( manufacturer=manufacturer, model='Test Performance Switch', defaults={'slug': 'test-perf-switch'} ) role, _ = DeviceRole.objects.get_or_create( name='Test Performance Role', defaults={'slug': 'test-perf-role'} ) # Calculate how many devices we need (48 interfaces per device) interfaces_per_device = 48 num_devices = (count + interfaces_per_device - 1) // interfaces_per_device self.stdout.write(f"Creating {num_devices} devices...") # Create devices devices = [] for i in range(num_devices): devices.append(Device( name=f'test-perf-device-{i:05d}', device_type=device_type, role=role, site=site )) # Bulk create devices start_time = time.time() with transaction.atomic(): Device.objects.bulk_create(devices, batch_size=1000) device_creation_time = time.time() - start_time self.stdout.write(self.style.SUCCESS(f" Created {num_devices} devices in {device_creation_time:.2f}s")) # Fetch created devices back (we need their PKs) devices = list(Device.objects.filter(name__startswith='test-perf-device-').order_by('name')) # Create interfaces self.stdout.write(f"Creating {count} interfaces...") interfaces = [] interface_types = [ InterfaceTypeChoices.TYPE_1GE_FIXED, InterfaceTypeChoices.TYPE_10GE_FIXED, InterfaceTypeChoices.TYPE_25GE_SR, InterfaceTypeChoices.TYPE_40GE_SR4, InterfaceTypeChoices.TYPE_100GE_SR4, ] created_count = 0 device_idx = 0 port_num = 1 start_time = time.time() for i in range(count): if port_num > interfaces_per_device: device_idx += 1 port_num = 1 if device_idx >= len(devices): break interface = Interface( device=devices[device_idx], name=f'eth{port_num}', type=interface_types[i % len(interface_types)], enabled=True ) interfaces.append(interface) port_num += 1 created_count += 1 # Show progress every 1000 interfaces if created_count % 1000 == 0: self.draw_progress_bar(created_count, count, 'Creating interfaces') self.draw_progress_bar(created_count, count, 'Creating interfaces') self.stdout.write() # newline # Bulk create interfaces in batches with transaction.atomic(): Interface.objects.bulk_create(interfaces, batch_size=1000) interface_creation_time = time.time() - start_time self.stdout.write(self.style.SUCCESS(f" Created {created_count} interfaces in {interface_creation_time:.2f}s")) # Fetch created interfaces to set up relationships all_interfaces = list(Interface.objects.filter(device__in=devices).order_by('id')) if len(all_interfaces) < 100: self.stdout.write(self.style.WARNING(" Not enough interfaces to create relationships")) return # Create LAG interfaces and assign members self.stdout.write("Creating LAG relationships...") num_lags = min(100, len(all_interfaces) // 10) lag_updates = [] for i in range(num_lags): # Every 10th interface becomes a LAG lag_interface = all_interfaces[i * 10] lag_interface.type = InterfaceTypeChoices.TYPE_LAG lag_updates.append(lag_interface) # Next 3 interfaces become LAG members for j in range(1, 4): idx = i * 10 + j if idx < len(all_interfaces): member = all_interfaces[idx] member.lag = lag_interface lag_updates.append(member) with transaction.atomic(): Interface.objects.bulk_update(lag_updates, ['type', 'lag'], batch_size=500) self.stdout.write(self.style.SUCCESS(f" Created {num_lags} LAG interfaces with members")) # Create bridge relationships self.stdout.write("Creating bridge relationships...") bridge_updates = [] num_bridges = min(50, len(all_interfaces) // 20) for i in range(num_bridges): # Create bridge interfaces bridge_idx = i * 20 + 5 if bridge_idx >= len(all_interfaces): break bridge_interface = all_interfaces[bridge_idx] bridge_interface.type = InterfaceTypeChoices.TYPE_BRIDGE bridge_updates.append(bridge_interface) # Bridge 2 interfaces to it for j in range(1, 3): member_idx = bridge_idx + j if member_idx < len(all_interfaces): member = all_interfaces[member_idx] member.bridge = bridge_interface bridge_updates.append(member) with transaction.atomic(): Interface.objects.bulk_update(bridge_updates, ['type', 'bridge'], batch_size=500) self.stdout.write(self.style.SUCCESS(f" Created {num_bridges} bridge interfaces")) # Create parent-child relationships self.stdout.write("Creating parent-child relationships (subinterfaces)...") parent_updates = [] num_parents = min(100, len(all_interfaces) // 15) for i in range(num_parents): parent_idx = i * 15 if parent_idx >= len(all_interfaces): break parent = all_interfaces[parent_idx] # Create 2-3 subinterfaces for j in range(1, 4): child_idx = parent_idx + j + 10 if child_idx < len(all_interfaces): child = all_interfaces[child_idx] child.parent = parent # Change name to subinterface format child.name = f'{parent.name}.{j}' parent_updates.append(child) with transaction.atomic(): Interface.objects.bulk_update(parent_updates, ['parent', 'name'], batch_size=500) self.stdout.write( self.style.SUCCESS(f" Created parent-child relationships for {len(parent_updates)} interfaces") ) total_time = device_creation_time + interface_creation_time self.stdout.write() self.stdout.write(self.style.SUCCESS("=" * 70)) self.stdout.write(self.style.SUCCESS("Test data creation complete!")) self.stdout.write(self.style.SUCCESS(f"Total time: {total_time:.2f}s")) self.stdout.write() self.stdout.write("Created:") self.stdout.write(f" - {created_count} total interfaces") self.stdout.write(f" - {num_lags} LAG interfaces with members") self.stdout.write(f" - {num_bridges} bridge interfaces") self.stdout.write(f" - {len(parent_updates)} subinterfaces with parents") self.stdout.write() self.stdout.write("Now visit /dcim/interfaces/ in your browser to test page load time.") self.stdout.write() self.stdout.write(f"With {created_count:,} interfaces, the performance issue should be visible.") self.stdout.write("With the fix applied, the page should load normally.") self.stdout.write("Without the fix (v4.5.0), it would timeout or take a long time to load.") self.stdout.write("=" * 70) def handle(self, *args, **options): count = options['count'] if count < 50000: self.stdout.write(self.style.WARNING( f"Warning: {count} interfaces may not show significant performance difference." )) self.stdout.write("The issue becomes apparent with 100k+ interfaces.") self.stdout.write("Consider using --count=100000 or higher for best demonstration.") if not options['no_input']: confirmation = input("Continue anyway? (yes/no): ") if confirmation != 'yes': self.stdout.write(self.style.SUCCESS("Aborting.")) return self.create_test_data(count, options) ```

MrUnknownDE closed this issue

2026-04-05 16:42:35 +02:00

Sign in to join this conversation.

Branches Tags

main

21455-sql-indexes-audit

21357-register-model-actions

feature

20924-plugin-ui-components

21766-improve-test-coverage-for-sortable-table-columns

21157-export-template-public-models

21025-pre-render-config-contexts

21364-swagger

feature-ip-prefix-link

20911-dropdown-3

fix_module_substitution

21160-filterset

20911-dropdown-2

20044-elevation-stuck-lightmode

7604-filter-modifiers-v3

20660-script-load

19724-graphql

14884-script

19720-macaddress-interface-generic-relation

fix-19669-api-image-download

7604-filter-modifiers

19275-fixes-interface-bulk-edit

fix-17794-get_field_value_return_list

11507-show-aggregate-and-rir-on-api

9583-add_column_specific_search_field_to_tables

No Label

1 Participants

Notifications

Due Date

No due date set.

Dependencies

No dependencies set.

Reference: github/netbox#546