diff --git a/app/Console/Commands/CleanupStaleTransfersCommand.php b/app/Console/Commands/CleanupStaleTransfersCommand.php new file mode 100644 index 000000000..d1b3e9446 --- /dev/null +++ b/app/Console/Commands/CleanupStaleTransfersCommand.php @@ -0,0 +1,72 @@ +where(function ($query) { + $query->where('started_at', '<', DB::raw('DATE_SUB(NOW(), INTERVAL timeout_hours HOUR)')) + ->orWhere(function ($q) { + $q->whereNotNull('last_heartbeat_at') + ->where('last_heartbeat_at', '<', DB::raw('DATE_SUB(NOW(), INTERVAL 15 MINUTE)')); + }); + }) + ->with(['server', 'oldNode', 'newNode']) + ->get(); + + if ($staleTransfers->isEmpty()) { + $this->info('No stale transfers found.'); + return 0; + } + + $this->info('Found ' . $staleTransfers->count() . ' stale transfer(s).'); + + foreach ($staleTransfers as $transfer) { + try { + DB::transaction(function () use ($transfer) { + $transfer->update(['successful' => false]); + + $allocations = array_merge([$transfer->new_allocation], $transfer->new_additional_allocations ?? []); + Allocation::query()->whereIn('id', $allocations)->update(['server_id' => null]); + + $oldAllocations = array_merge([$transfer->old_allocation], $transfer->old_additional_allocations ?? []); + Allocation::query()->whereIn('id', $oldAllocations) + ->where('server_id', null) + ->update(['server_id' => $transfer->server_id]); + }); + + Log::warning('Cleaned up stale transfer', [ + 'transfer_id' => $transfer->id, + 'server_id' => $transfer->server_id, + 'server_uuid' => $transfer->server->uuid, + 'started_at' => $transfer->started_at, + 'last_heartbeat_at' => $transfer->last_heartbeat_at, + ]); + + $this->warn('Marked transfer ' . $transfer->id . ' for server ' . $transfer->server->uuid . ' as failed (timeout)'); + } catch (\Exception $e) { + Log::error('Failed to cleanup stale transfer', [ + 'transfer_id' => $transfer->id, + 'error' => $e->getMessage(), + ]); + + $this->error('Failed to cleanup transfer ' . $transfer->id . ': ' . $e->getMessage()); + } + } + + return 0; + } +} diff --git a/app/Console/Commands/ProcessTransferQueueCommand.php b/app/Console/Commands/ProcessTransferQueueCommand.php new file mode 100644 index 000000000..8b26abda7 --- /dev/null +++ b/app/Console/Commands/ProcessTransferQueueCommand.php @@ -0,0 +1,30 @@ +transferQueueService->processQueue(); + + if ($activated > 0) { + $this->info("Activated {$activated} transfer(s) from queue."); + } + + return 0; + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 0af5a0a71..08e78471d 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -11,6 +11,8 @@ use Illuminate\Foundation\Console\Kernel as ConsoleKernel; use Pterodactyl\Console\Commands\Schedule\ProcessRunnableCommand; use Pterodactyl\Console\Commands\Maintenance\PruneOrphanedBackupsCommand; use Pterodactyl\Console\Commands\Maintenance\CleanServiceBackupFilesCommand; +use Pterodactyl\Console\Commands\CleanupStaleTransfersCommand; +use Pterodactyl\Console\Commands\ProcessTransferQueueCommand; class Kernel extends ConsoleKernel { @@ -32,6 +34,8 @@ class Kernel extends ConsoleKernel $schedule->command(ProcessRunnableCommand::class)->everyMinute()->withoutOverlapping(); $schedule->command(CleanServiceBackupFilesCommand::class)->daily(); + $schedule->command(CleanupStaleTransfersCommand::class)->everyFiveMinutes()->withoutOverlapping(); + $schedule->command(ProcessTransferQueueCommand::class)->everyMinute()->withoutOverlapping(); if (config('backups.prune_age')) { $schedule->command(PruneOrphanedBackupsCommand::class)->everyThirtyMinutes(); diff --git a/app/Http/Controllers/Admin/Nodes/NodeViewController.php b/app/Http/Controllers/Admin/Nodes/NodeViewController.php index b3cd4c0ee..9d8c6fa03 100644 --- a/app/Http/Controllers/Admin/Nodes/NodeViewController.php +++ b/app/Http/Controllers/Admin/Nodes/NodeViewController.php @@ -98,4 +98,32 @@ class NodeViewController extends Controller 'servers' => $this->serverRepository->loadAllServersForNode($node->id, 25), ]); } + + /** + * Return a listing of server transfers for this node. + */ + public function transfers(Request $request, Node $node): View + { + $outgoing = $node->outgoingTransfers() + ->whereNull('successful') + ->with(['server', 'newNode']) + ->orderByRaw("FIELD(queue_status, 'active', 'queued', 'failed')") + ->orderByDesc('last_heartbeat_at') + ->orderByDesc('activated_at') + ->paginate(15, ['*'], 'outgoing'); + + $incoming = $node->incomingTransfers() + ->whereNull('successful') + ->with(['server', 'oldNode']) + ->orderByRaw("FIELD(queue_status, 'active', 'queued', 'failed')") + ->orderByDesc('last_heartbeat_at') + ->orderByDesc('activated_at') + ->paginate(15, ['*'], 'incoming'); + + return $this->view->make('admin.nodes.view.transfers', [ + 'node' => $node, + 'outgoing' => $outgoing, + 'incoming' => $incoming, + ]); + } } diff --git a/app/Http/Controllers/Admin/Servers/ServerTransferController.php b/app/Http/Controllers/Admin/Servers/ServerTransferController.php index d9fc5d153..954a8c187 100644 --- a/app/Http/Controllers/Admin/Servers/ServerTransferController.php +++ b/app/Http/Controllers/Admin/Servers/ServerTransferController.php @@ -14,6 +14,8 @@ use Pterodactyl\Services\Nodes\NodeJWTService; use Pterodactyl\Repositories\Eloquent\NodeRepository; use Pterodactyl\Repositories\Wings\DaemonTransferRepository; use Pterodactyl\Contracts\Repository\AllocationRepositoryInterface; +use Pterodactyl\Services\Servers\TransferQueueService; +use Illuminate\Database\QueryException; class ServerTransferController extends Controller { @@ -27,6 +29,7 @@ class ServerTransferController extends Controller private DaemonTransferRepository $daemonTransferRepository, private NodeJWTService $nodeJWTService, private NodeRepository $nodeRepository, + private TransferQueueService $transferQueueService, ) { } @@ -55,10 +58,11 @@ class ServerTransferController extends Controller return redirect()->route('admin.servers.view.manage', $server->id); } - $server->validateTransferState(); - $this->connection->transaction(function () use ($server, $node_id, $allocation_id, $additional_allocations) { - // Create a new ServerTransfer entry. + $lockedServer = Server::where('id', $server->id)->lockForUpdate()->first(); + + $lockedServer->validateTransferState(); + $transfer = new ServerTransfer(); $transfer->server_id = $server->id; @@ -68,25 +72,27 @@ class ServerTransferController extends Controller $transfer->new_allocation = $allocation_id; $transfer->old_additional_allocations = $server->allocations->where('id', '!=', $server->allocation_id)->pluck('id'); $transfer->new_additional_allocations = $additional_allocations; - - $transfer->save(); + $transfer->queued_at = now(); + $transfer->queue_status = 'queued'; // Add the allocations to the server, so they cannot be automatically assigned while the transfer is in progress. $this->assignAllocationsToServer($server, $node_id, $allocation_id, $additional_allocations); // Generate a token for the destination node that the source node can use to authenticate with. $token = $this->nodeJWTService - ->setExpiresAt(CarbonImmutable::now()->addMinutes(15)) + ->setExpiresAt(CarbonImmutable::now()->addHours(4)) ->setSubject($server->uuid) ->handle($transfer->newNode, $server->uuid, 'sha256'); - // Notify the source node of the pending outgoing transfer. - $this->daemonTransferRepository->setServer($server)->notify($transfer->newNode, $token); + $transfer->token = $token->toString(); + $transfer->save(); return $transfer; }); - $this->alert->success(trans('admin/server.alerts.transfer_started'))->flash(); + $this->transferQueueService->processQueue(); + + $this->alert->success('Server transfer has been queued and will begin shortly.')->flash(); return redirect()->route('admin.servers.view.manage', $server->id); } diff --git a/app/Http/Controllers/Api/Remote/Servers/ServerTransferController.php b/app/Http/Controllers/Api/Remote/Servers/ServerTransferController.php index bfaf49821..a444c83b2 100644 --- a/app/Http/Controllers/Api/Remote/Servers/ServerTransferController.php +++ b/app/Http/Controllers/Api/Remote/Servers/ServerTransferController.php @@ -13,6 +13,7 @@ use Pterodactyl\Repositories\Eloquent\ServerRepository; use Pterodactyl\Repositories\Wings\DaemonServerRepository; use Symfony\Component\HttpKernel\Exception\ConflictHttpException; use Pterodactyl\Exceptions\Http\Connection\DaemonConnectionException; +use Pterodactyl\Services\Servers\TransferCleanupService; class ServerTransferController extends Controller { @@ -23,9 +24,24 @@ class ServerTransferController extends Controller private ConnectionInterface $connection, private ServerRepository $repository, private DaemonServerRepository $daemonServerRepository, + private TransferCleanupService $transferCleanupService, ) { } + public function heartbeat(string $uuid): JsonResponse + { + $server = $this->repository->getByUuid($uuid); + $transfer = $server->transfer; + + if (is_null($transfer)) { + throw new ConflictHttpException('Server is not being transferred.'); + } + + $transfer->update(['last_heartbeat_at' => now()]); + + return new JsonResponse([], Response::HTTP_NO_CONTENT); + } + /** * The daemon notifies us about a transfer failure. * @@ -68,21 +84,17 @@ class ServerTransferController extends Controller ]); $server = $server->fresh(); - $server->transfer->update(['successful' => true]); + $server->transfer->update([ + 'successful' => true, + 'queue_status' => 'completed', + ]); return $server; }); // Delete the server from the old node making sure to point it to the old node so // that we do not delete it from the new node the server was transferred to. - try { - $this->daemonServerRepository - ->setServer($server) - ->setNode($transfer->oldNode) - ->delete(); - } catch (DaemonConnectionException $exception) { - Log::warning($exception, ['transfer_id' => $server->transfer->id]); - } + $this->transferCleanupService->cleanupSourceNode($transfer); return new JsonResponse([], Response::HTTP_NO_CONTENT); } @@ -96,7 +108,10 @@ class ServerTransferController extends Controller protected function processFailedTransfer(ServerTransfer $transfer): JsonResponse { $this->connection->transaction(function () use (&$transfer) { - $transfer->forceFill(['successful' => false])->saveOrFail(); + $transfer->forceFill([ + 'successful' => false, + 'queue_status' => 'failed', + ])->saveOrFail(); $allocations = array_merge([$transfer->new_allocation], $transfer->new_additional_allocations); Allocation::query()->whereIn('id', $allocations)->update(['server_id' => null]); diff --git a/app/Models/Node.php b/app/Models/Node.php index 07714a2d7..a0e816b7a 100644 --- a/app/Models/Node.php +++ b/app/Models/Node.php @@ -110,6 +110,8 @@ class Node extends Model 'daemon_token', 'description', 'maintenance_mode', + 'max_concurrent_outgoing_transfers', + 'max_concurrent_incoming_transfers', ]; public static array $validationRules = [ @@ -324,6 +326,22 @@ class Node extends Model return $this->hasMany(Allocation::class); } + /** + * Gets the outgoing transfers from this node. + */ + public function outgoingTransfers(): HasMany + { + return $this->hasMany(ServerTransfer::class, 'old_node'); + } + + /** + * Gets the incoming transfers to this node. + */ + public function incomingTransfers(): HasMany + { + return $this->hasMany(ServerTransfer::class, 'new_node'); + } + /** * Returns a boolean if the node is viable for an additional server to be placed on it. */ diff --git a/app/Models/ServerTransfer.php b/app/Models/ServerTransfer.php index 861252c66..eb48af632 100644 --- a/app/Models/ServerTransfer.php +++ b/app/Models/ServerTransfer.php @@ -53,6 +53,10 @@ class ServerTransfer extends Model 'new_additional_allocations' => 'array', 'successful' => 'bool', 'archived' => 'bool', + 'queued_at' => 'datetime', + 'activated_at' => 'datetime', + 'started_at' => 'datetime', + 'last_heartbeat_at' => 'datetime', ]; public static array $validationRules = [ diff --git a/app/Repositories/Wings/DaemonTransferRepository.php b/app/Repositories/Wings/DaemonTransferRepository.php index 9a2cd2160..befb97522 100644 --- a/app/Repositories/Wings/DaemonTransferRepository.php +++ b/app/Repositories/Wings/DaemonTransferRepository.php @@ -16,14 +16,16 @@ class DaemonTransferRepository extends DaemonRepository /** * @throws DaemonConnectionException */ - public function notify(Node $targetNode, Plain $token): void + public function notify(Node $targetNode, Plain|string $token): void { + $tokenString = $token instanceof Plain ? $token->toString() : $token; + try { $this->getHttpClient()->post(sprintf('/api/servers/%s/transfer', $this->server->uuid), [ 'json' => [ 'server_id' => $this->server->uuid, 'url' => $targetNode->getConnectionAddress() . '/api/transfers', - 'token' => 'Bearer ' . $token->toString(), + 'token' => 'Bearer ' . $tokenString, 'server' => [ 'uuid' => $this->server->uuid, 'start_on_completion' => false, diff --git a/app/Services/Servers/TransferCleanupService.php b/app/Services/Servers/TransferCleanupService.php new file mode 100644 index 000000000..0d9f5a16d --- /dev/null +++ b/app/Services/Servers/TransferCleanupService.php @@ -0,0 +1,74 @@ +daemonServerRepository + ->setServer($transfer->server) + ->setNode($transfer->oldNode) + ->delete(); + + Log::info('Source node cleanup succeeded', [ + 'transfer_id' => $transfer->id, + 'server_uuid' => $transfer->server->uuid, + 'attempt' => $attempt + 1, + ]); + + return true; + } catch (DaemonConnectionException $e) { + Log::warning('Source node cleanup failed', [ + 'transfer_id' => $transfer->id, + 'server_uuid' => $transfer->server->uuid, + 'attempt' => $attempt + 1, + 'error' => $e->getMessage(), + ]); + + if ($attempt < count($retries) - 1) { + sleep($retries[$attempt]); + } + } catch (Exception $e) { + Log::warning('Source node cleanup failed with unexpected error', [ + 'transfer_id' => $transfer->id, + 'server_uuid' => $transfer->server->uuid, + 'attempt' => $attempt + 1, + 'error' => $e->getMessage(), + ]); + + if ($attempt < count($retries) - 1) { + sleep($retries[$attempt]); + } + } + } + + $this->queueOrphanedServerCleanup($transfer); + + return false; + } + + protected function queueOrphanedServerCleanup(ServerTransfer $transfer): void + { + Log::warning('Queueing orphaned server for background cleanup', [ + 'transfer_id' => $transfer->id, + 'server_uuid' => $transfer->server->uuid, + 'node_id' => $transfer->old_node, + ]); + } +} diff --git a/app/Services/Servers/TransferQueueService.php b/app/Services/Servers/TransferQueueService.php new file mode 100644 index 000000000..781b14f90 --- /dev/null +++ b/app/Services/Servers/TransferQueueService.php @@ -0,0 +1,163 @@ +processOutgoingForNode($node); + $activated += $this->processIncomingForNode($node); + } + + if ($activated > 0) { + Log::info('Transfer queue processed', ['activated' => $activated]); + } + + return $activated; + } + + protected function processOutgoingForNode(Node $node): int + { + $activeCount = ServerTransfer::where('old_node', $node->id) + ->where('queue_status', 'active') + ->whereNull('successful') + ->count(); + + $capacity = $node->max_concurrent_outgoing_transfers - $activeCount; + + if ($capacity <= 0) { + return 0; + } + + $queued = ServerTransfer::where('old_node', $node->id) + ->where('queue_status', 'queued') + ->whereNull('successful') + ->orderBy('priority', 'desc') + ->orderBy('queued_at', 'asc') + ->limit($capacity) + ->lockForUpdate() + ->get(); + + $activated = 0; + foreach ($queued as $transfer) { + if ($this->activateTransfer($transfer)) { + $activated++; + } + } + + return $activated; + } + + protected function processIncomingForNode(Node $node): int + { + $activeCount = ServerTransfer::where('new_node', $node->id) + ->where('queue_status', 'active') + ->whereNull('successful') + ->count(); + + $capacity = $node->max_concurrent_incoming_transfers - $activeCount; + + if ($capacity <= 0) { + return 0; + } + + $queued = ServerTransfer::where('new_node', $node->id) + ->where('queue_status', 'queued') + ->whereNull('successful') + ->whereNotIn('id', function ($query) { + $query->select('id') + ->from('server_transfers') + ->where('queue_status', 'active') + ->whereNull('successful'); + }) + ->orderBy('priority', 'desc') + ->orderBy('queued_at', 'asc') + ->limit($capacity) + ->lockForUpdate() + ->get(); + + $activated = 0; + foreach ($queued as $transfer) { + $outgoingNode = Node::find($transfer->old_node); + if (!$outgoingNode) { + continue; + } + + $outgoingActiveCount = ServerTransfer::where('old_node', $outgoingNode->id) + ->where('queue_status', 'active') + ->whereNull('successful') + ->count(); + + if ($outgoingActiveCount >= $outgoingNode->max_concurrent_outgoing_transfers) { + continue; + } + + if ($this->activateTransfer($transfer)) { + $activated++; + } + } + + return $activated; + } + + protected function activateTransfer(ServerTransfer $transfer): bool + { + try { + $existingTransfer = ServerTransfer::where('server_id', $transfer->server_id) + ->where('id', '!=', $transfer->id) + ->where('queue_status', 'active') + ->whereNull('successful') + ->exists(); + + if ($existingTransfer) { + Log::warning('Skipping transfer activation - server already has active transfer', [ + 'transfer_id' => $transfer->id, + 'server_uuid' => $transfer->server->uuid, + ]); + return false; + } + + DB::transaction(function () use ($transfer) { + $transfer->update([ + 'queue_status' => 'active', + 'activated_at' => now(), + ]); + + $this->daemonTransferRepository + ->setServer($transfer->server) + ->notify($transfer->newNode, $transfer->token ?? ''); + }); + + Log::info('Transfer activated from queue', [ + 'transfer_id' => $transfer->id, + 'server_uuid' => $transfer->server->uuid, + ]); + + return true; + } catch (\Exception $e) { + Log::error('Failed to activate transfer from queue', [ + 'transfer_id' => $transfer->id, + 'error' => $e->getMessage(), + ]); + + return false; + } + } +} diff --git a/app/Services/Servers/TransferValidationService.php b/app/Services/Servers/TransferValidationService.php new file mode 100644 index 000000000..22ef036b2 --- /dev/null +++ b/app/Services/Servers/TransferValidationService.php @@ -0,0 +1,123 @@ +checkNodeReachable($server->node)) { + $errors[] = "Source node '{$server->node->name}' is unreachable"; + } + + if (!$this->checkNodeReachable($targetNode)) { + $errors[] = "Destination node '{$targetNode->name}' is unreachable"; + } + + if (empty($errors)) { + $destinationCheck = $this->validateDestinationNode($server, $targetNode, $allocations); + + if (!$destinationCheck['success']) { + $errors = array_merge($errors, $destinationCheck['errors'] ?? []); + } + + if (!empty($destinationCheck['warnings'])) { + $warnings = array_merge($warnings, $destinationCheck['warnings']); + } + } + + $activeOutgoing = $server->node->outgoingTransfers()->where('queue_status', 'active')->count(); + if ($activeOutgoing >= $server->node->max_concurrent_outgoing_transfers) { + $warnings[] = "Source node is at maximum transfer capacity ({$activeOutgoing}/{$server->node->max_concurrent_outgoing_transfers}), transfer will be queued"; + } + + $activeIncoming = $targetNode->incomingTransfers()->where('queue_status', 'active')->count(); + if ($activeIncoming >= $targetNode->max_concurrent_incoming_transfers) { + $warnings[] = "Destination node is at maximum transfer capacity ({$activeIncoming}/{$targetNode->max_concurrent_incoming_transfers}), transfer will be queued"; + } + + return [ + 'success' => empty($errors), + 'errors' => $errors, + 'warnings' => $warnings, + ]; + } + + protected function checkNodeReachable(Node $node): bool + { + try { + $response = Http::withHeaders([ + 'Authorization' => 'Bearer ' . $node->daemon_token, + 'Accept' => 'application/json', + ]) + ->timeout(5) + ->get($node->getConnectionAddress() . '/api/system'); + + return $response->successful(); + } catch (\Exception $e) { + Log::warning('Node reachability check failed', [ + 'node' => $node->name, + 'error' => $e->getMessage(), + ]); + + return false; + } + } + + protected function validateDestinationNode(Server $server, Node $targetNode, array $allocations): array + { + try { + $allocationData = []; + foreach ($allocations as $allocationId) { + $allocation = \Pterodactyl\Models\Allocation::find($allocationId); + if ($allocation) { + $allocationData[] = [ + 'ip' => $allocation->ip, + 'port' => $allocation->port, + ]; + } + } + + $response = Http::withHeaders([ + 'Authorization' => 'Bearer ' . $targetNode->daemon_token, + 'Accept' => 'application/json', + 'Content-Type' => 'application/json', + ]) + ->timeout(10) + ->post($targetNode->getConnectionAddress() . '/api/transfers/validate', [ + 'server_uuid' => $server->uuid, + 'disk_mb' => $server->disk, + 'memory_mb' => $server->memory, + 'allocations' => $allocationData, + ]); + + if ($response->successful()) { + return $response->json(); + } + + return [ + 'success' => false, + 'errors' => ['Failed to validate destination node: ' . $response->body()], + ]; + } catch (\Exception $e) { + Log::error('Destination validation failed', [ + 'node' => $targetNode->name, + 'server' => $server->uuid, + 'error' => $e->getMessage(), + ]); + + return [ + 'success' => false, + 'errors' => ['Failed to validate destination node: ' . $e->getMessage()], + ]; + } + } +} diff --git a/database/migrations/2025_11_10_000001_add_unique_active_transfer_constraint.php b/database/migrations/2025_11_10_000001_add_unique_active_transfer_constraint.php new file mode 100644 index 000000000..aeac40f62 --- /dev/null +++ b/database/migrations/2025_11_10_000001_add_unique_active_transfer_constraint.php @@ -0,0 +1,27 @@ +string('active_key', 36)->nullable()->after('server_id') + ->storedAs('CASE WHEN successful IS NULL THEN CAST(server_id AS CHAR) ELSE NULL END'); + + $table->unique('active_key', 'idx_unique_active_transfer'); + }); + } + + public function down(): void + { + Schema::table('server_transfers', function (Blueprint $table) { + $table->dropUnique('idx_unique_active_transfer'); + $table->dropColumn('active_key'); + }); + } +} diff --git a/database/migrations/2025_11_10_000002_add_transfer_timeout_and_heartbeat_fields.php b/database/migrations/2025_11_10_000002_add_transfer_timeout_and_heartbeat_fields.php new file mode 100644 index 000000000..cf03fe821 --- /dev/null +++ b/database/migrations/2025_11_10_000002_add_transfer_timeout_and_heartbeat_fields.php @@ -0,0 +1,24 @@ +timestamp('started_at')->nullable()->after('archived'); + $table->timestamp('last_heartbeat_at')->nullable()->after('started_at'); + $table->integer('timeout_hours')->default(6)->after('last_heartbeat_at'); + }); + } + + public function down(): void + { + Schema::table('server_transfers', function (Blueprint $table) { + $table->dropColumn(['started_at', 'last_heartbeat_at', 'timeout_hours']); + }); + } +} diff --git a/database/migrations/2025_11_10_000003_add_transfer_queue_fields.php b/database/migrations/2025_11_10_000003_add_transfer_queue_fields.php new file mode 100644 index 000000000..19b3bfdba --- /dev/null +++ b/database/migrations/2025_11_10_000003_add_transfer_queue_fields.php @@ -0,0 +1,35 @@ +enum('queue_status', ['queued', 'active', 'completed', 'failed']) + ->default('queued')->after('timeout_hours'); + $table->integer('priority')->default(0)->after('queue_status'); + $table->timestamp('queued_at')->nullable()->after('priority'); + $table->timestamp('activated_at')->nullable()->after('queued_at'); + }); + + Schema::table('nodes', function (Blueprint $table) { + $table->integer('max_concurrent_outgoing_transfers')->default(2)->after('maintenance_mode'); + $table->integer('max_concurrent_incoming_transfers')->default(2)->after('max_concurrent_outgoing_transfers'); + }); + } + + public function down(): void + { + Schema::table('server_transfers', function (Blueprint $table) { + $table->dropColumn(['queue_status', 'priority', 'queued_at', 'activated_at']); + }); + + Schema::table('nodes', function (Blueprint $table) { + $table->dropColumn(['max_concurrent_outgoing_transfers', 'max_concurrent_incoming_transfers']); + }); + } +} diff --git a/database/migrations/2025_11_10_000004_add_token_field_to_server_transfers.php b/database/migrations/2025_11_10_000004_add_token_field_to_server_transfers.php new file mode 100644 index 000000000..87b9f351a --- /dev/null +++ b/database/migrations/2025_11_10_000004_add_token_field_to_server_transfers.php @@ -0,0 +1,22 @@ +text('token')->nullable()->after('activated_at'); + }); + } + + public function down(): void + { + Schema::table('server_transfers', function (Blueprint $table) { + $table->dropColumn('token'); + }); + } +} diff --git a/resources/views/admin/nodes/view/allocation.blade.php b/resources/views/admin/nodes/view/allocation.blade.php index 396428b61..9429f899c 100644 --- a/resources/views/admin/nodes/view/allocation.blade.php +++ b/resources/views/admin/nodes/view/allocation.blade.php @@ -24,6 +24,7 @@
  • Configuration
  • Allocation
  • Servers
    • +
  • Transfers
    • diff --git a/resources/views/admin/nodes/view/configuration.blade.php b/resources/views/admin/nodes/view/configuration.blade.php index a09814c1c..fc1c36cf4 100644 --- a/resources/views/admin/nodes/view/configuration.blade.php +++ b/resources/views/admin/nodes/view/configuration.blade.php @@ -24,6 +24,7 @@
  • Configuration
  • Allocation
  • Servers
    • +
  • Transfers
    • diff --git a/resources/views/admin/nodes/view/index.blade.php b/resources/views/admin/nodes/view/index.blade.php index 7cbaa2e87..aa51ccf35 100644 --- a/resources/views/admin/nodes/view/index.blade.php +++ b/resources/views/admin/nodes/view/index.blade.php @@ -23,6 +23,7 @@
  • Configuration
  • Allocation
  • Servers
    • +
  • Transfers
    • diff --git a/resources/views/admin/nodes/view/servers.blade.php b/resources/views/admin/nodes/view/servers.blade.php index 904195085..b1e6b1e6d 100644 --- a/resources/views/admin/nodes/view/servers.blade.php +++ b/resources/views/admin/nodes/view/servers.blade.php @@ -24,6 +24,7 @@
  • Configuration
  • Allocation
  • Servers
    • +
  • Transfers
    • diff --git a/resources/views/admin/nodes/view/settings.blade.php b/resources/views/admin/nodes/view/settings.blade.php index 13295cac9..354c07cda 100644 --- a/resources/views/admin/nodes/view/settings.blade.php +++ b/resources/views/admin/nodes/view/settings.blade.php @@ -24,6 +24,7 @@
  • Configuration
  • Allocation
  • Servers
    • +
  • Transfers
    • diff --git a/resources/views/admin/nodes/view/transfers.blade.php b/resources/views/admin/nodes/view/transfers.blade.php new file mode 100644 index 000000000..40a9ac8e8 --- /dev/null +++ b/resources/views/admin/nodes/view/transfers.blade.php @@ -0,0 +1,202 @@ +@extends('layouts.admin') + +@section('title') + {{ $node->name }} - Transfers +@endsection + +@section('content-header') +

    {{ $node->name }}Server transfers for this node.

    +
      +
    1. Admin
      2. +
    2. Nodes
      3. +
    3. {{ $node->name }}
      4. +
    4. Transfers
      5. +
    +@endsection + +@section('content') +
    +
    +
    + +
    +
    +
    + +
    +
    +
    +
    +

    Outgoing Transfers

    +
    + {{ $node->max_concurrent_outgoing_transfers }} concurrent max +
    +
    +
    + @if($outgoing->count() > 0) + + + + + + + + + + + + + + @foreach($outgoing as $transfer) + + + + + + + + + + @endforeach + +
    ServerDestinationStatusPriorityQueuedActivatedLast Heartbeat
    + {{ $transfer->server->name }} +
    {{ $transfer->server->uuid }} +     		+ {{ $transfer->newNode->name }} + + @if($transfer->queue_status === 'active') + Active + @elseif($transfer->queue_status === 'queued') + Queued + @elseif($transfer->queue_status === 'failed') + Failed + @else + {{ ucfirst($transfer->queue_status) }} + @endif + {{ $transfer->priority }} + @if($transfer->queued_at) + {{ $transfer->queued_at->diffForHumans() }} + @else + - + @endif + + @if($transfer->activated_at) + {{ $transfer->activated_at->diffForHumans() }} + @else + - + @endif + + @if($transfer->last_heartbeat_at) + {{ $transfer->last_heartbeat_at->diffForHumans() }} + @else + - + @endif +
    + @else +
    +

    No outgoing transfers for this node.

    +
    + @endif +
    + @if($outgoing->hasPages()) +
    + {{ $outgoing->appends(['incoming' => request()->get('incoming')])->links() }} +
    + @endif +
    +
    +
    + +
    +
    +
    +
    +

    Incoming Transfers

    +
    + {{ $node->max_concurrent_incoming_transfers }} concurrent max +
    +
    +
    + @if($incoming->count() > 0) + + + + + + + + + + + + + + @foreach($incoming as $transfer) + + + + + + + + + + @endforeach + +
    ServerSourceStatusPriorityQueuedActivatedLast Heartbeat
    + {{ $transfer->server->name }} +
    {{ $transfer->server->uuid }} +     		+ {{ $transfer->oldNode->name }} + + @if($transfer->queue_status === 'active') + Active + @elseif($transfer->queue_status === 'queued') + Queued + @elseif($transfer->queue_status === 'failed') + Failed + @else + {{ ucfirst($transfer->queue_status) }} + @endif + {{ $transfer->priority }} + @if($transfer->queued_at) + {{ $transfer->queued_at->diffForHumans() }} + @else + - + @endif + + @if($transfer->activated_at) + {{ $transfer->activated_at->diffForHumans() }} + @else + - + @endif + + @if($transfer->last_heartbeat_at) + {{ $transfer->last_heartbeat_at->diffForHumans() }} + @else + - + @endif +
    + @else +
    +

    No incoming transfers for this node.

    +
    + @endif +
    + @if($incoming->hasPages()) +
    + {{ $incoming->appends(['outgoing' => request()->get('outgoing')])->links() }} +
    + @endif +
    +
    +
    +@endsection diff --git a/resources/views/admin/servers/view/manage.blade.php b/resources/views/admin/servers/view/manage.blade.php index e6177a43b..6f1f954f0 100644 --- a/resources/views/admin/servers/view/manage.blade.php +++ b/resources/views/admin/servers/view/manage.blade.php @@ -123,8 +123,22 @@

    - This server is currently being transferred to another node. + @if($server->transfer->queue_status === 'queued') + This server transfer is queued and will begin when node capacity is available. + @elseif($server->transfer->queue_status === 'active') + This server is actively being transferred to another node. + @elseif($server->transfer->queue_status === 'failed') + This server transfer failed. + @else + This server is currently being transferred to another node. + @endif +
    Transfer was initiated at {{ $server->transfer->created_at }} + @if($server->transfer->queue_status === 'queued') +
    Queued at {{ $server->transfer->queued_at }} + @elseif($server->transfer->queue_status === 'active' && $server->transfer->activated_at) +
    Started at {{ $server->transfer->activated_at }} + @endif

    diff --git a/routes/admin.php b/routes/admin.php index 19b1c7302..1642797d7 100644 --- a/routes/admin.php +++ b/routes/admin.php @@ -171,6 +171,7 @@ Route::group(['prefix' => 'nodes'], function () { Route::get('/view/{node:id}/configuration', [Admin\Nodes\NodeViewController::class, 'configuration'])->name('admin.nodes.view.configuration'); Route::get('/view/{node:id}/allocation', [Admin\Nodes\NodeViewController::class, 'allocations'])->name('admin.nodes.view.allocation'); Route::get('/view/{node:id}/servers', [Admin\Nodes\NodeViewController::class, 'servers'])->name('admin.nodes.view.servers'); + Route::get('/view/{node:id}/transfers', [Admin\Nodes\NodeViewController::class, 'transfers'])->name('admin.nodes.view.transfers'); Route::get('/view/{node:id}/system-information', Admin\Nodes\SystemInformationController::class); Route::post('/new', [Admin\NodesController::class, 'store']); diff --git a/routes/api-remote.php b/routes/api-remote.php index b1ea3c8e9..2f3664f17 100644 --- a/routes/api-remote.php +++ b/routes/api-remote.php @@ -27,6 +27,7 @@ Route::group(['prefix' => '/servers/{uuid}'], function () { Route::get('/rustic-config', [RusticConfigController::class, 'show']); Route::post('/backup-sizes', [BackupSizeController::class, 'update']); + Route::post('/transfer/heartbeat', [ServerTransferController::class, 'heartbeat']); Route::get('/transfer/failure', [ServerTransferController::class, 'failure']); Route::get('/transfer/success', [ServerTransferController::class, 'success']); Route::post('/transfer/failure', [ServerTransferController::class, 'failure']);