github/pyrodactyl
1
1
Fork 0
mirror of https://github.com/pyrohost/pyrodactyl.git synced 2026-04-05 19:51:59 +02:00
Code Issues 44 Packages Projects Releases Wiki Activity

feat: transfer queue

Browse Source
This commit is contained in:
Elizabeth
2025-11-10 15:38:29 -06:00
parent 7f8e4cac6d
commit 5f1e076dc9
25 changed files with 892 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
app/Console/Commands/CleanupStaleTransfersCommand.php Normal file
View File

@@ -0,0 +1,72 @@
<?php
namespace Pterodactyl\Console\Commands;
use Illuminate\Console\Command;
use Pterodactyl\Models\Allocation;
use Pterodactyl\Models\ServerTransfer;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
class CleanupStaleTransfersCommand extends Command
{
protected $signature = 'p:transfer:cleanup';
protected $description = 'Cleanup stale server transfers that have exceeded their timeout';
public function handle(): int
{
$staleTransfers = ServerTransfer::whereNull('successful')
->where(function ($query) {
$query->where('started_at', '<', DB::raw('DATE_SUB(NOW(), INTERVAL timeout_hours HOUR)'))
->orWhere(function ($q) {
$q->whereNotNull('last_heartbeat_at')
->where('last_heartbeat_at', '<', DB::raw('DATE_SUB(NOW(), INTERVAL 15 MINUTE)'));
});
})
->with(['server', 'oldNode', 'newNode'])
->get();
if ($staleTransfers->isEmpty()) {
$this->info('No stale transfers found.');
return 0;
}
$this->info('Found ' . $staleTransfers->count() . ' stale transfer(s).');
foreach ($staleTransfers as $transfer) {
try {
DB::transaction(function () use ($transfer) {
$transfer->update(['successful' => false]);
$allocations = array_merge([$transfer->new_allocation], $transfer->new_additional_allocations ?? []);
Allocation::query()->whereIn('id', $allocations)->update(['server_id' => null]);
$oldAllocations = array_merge([$transfer->old_allocation], $transfer->old_additional_allocations ?? []);
Allocation::query()->whereIn('id', $oldAllocations)
->where('server_id', null)
->update(['server_id' => $transfer->server_id]);
});
Log::warning('Cleaned up stale transfer', [
'transfer_id' => $transfer->id,
'server_id' => $transfer->server_id,
'server_uuid' => $transfer->server->uuid,
'started_at' => $transfer->started_at,
'last_heartbeat_at' => $transfer->last_heartbeat_at,
]);
$this->warn('Marked transfer ' . $transfer->id . ' for server ' . $transfer->server->uuid . ' as failed (timeout)');
} catch (\Exception $e) {
Log::error('Failed to cleanup stale transfer', [
'transfer_id' => $transfer->id,
'error' => $e->getMessage(),
]);
$this->error('Failed to cleanup transfer ' . $transfer->id . ': ' . $e->getMessage());
}
}
return 0;
}
}

30
app/Console/Commands/ProcessTransferQueueCommand.php Normal file
View File

@@ -0,0 +1,30 @@
<?php
namespace Pterodactyl\Console\Commands;
use Illuminate\Console\Command;
use Pterodactyl\Services\Servers\TransferQueueService;
class ProcessTransferQueueCommand extends Command
{
protected $signature = 'p:transfer:process-queue';
protected $description = 'Process the server transfer queue and activate waiting transfers';
public function __construct(
private TransferQueueService $transferQueueService,
) {
parent::__construct();
}
public function handle(): int
{
$activated = $this->transferQueueService->processQueue();
if ($activated > 0) {
$this->info("Activated {$activated} transfer(s) from queue.");
}
return 0;
}
}

4
app/Console/Kernel.php
View File

@@ -11,6 +11,8 @@ use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
use Pterodactyl\Console\Commands\Schedule\ProcessRunnableCommand;
use Pterodactyl\Console\Commands\Maintenance\PruneOrphanedBackupsCommand;
use Pterodactyl\Console\Commands\Maintenance\CleanServiceBackupFilesCommand;
use Pterodactyl\Console\Commands\CleanupStaleTransfersCommand;
use Pterodactyl\Console\Commands\ProcessTransferQueueCommand;
class Kernel extends ConsoleKernel
{
@@ -32,6 +34,8 @@ class Kernel extends ConsoleKernel
$schedule->command(ProcessRunnableCommand::class)->everyMinute()->withoutOverlapping();
$schedule->command(CleanServiceBackupFilesCommand::class)->daily();
$schedule->command(CleanupStaleTransfersCommand::class)->everyFiveMinutes()->withoutOverlapping();
$schedule->command(ProcessTransferQueueCommand::class)->everyMinute()->withoutOverlapping();
if (config('backups.prune_age')) {
$schedule->command(PruneOrphanedBackupsCommand::class)->everyThirtyMinutes();

28
app/Http/Controllers/Admin/Nodes/NodeViewController.php
View File

@@ -98,4 +98,32 @@ class NodeViewController extends Controller
'servers' => $this->serverRepository->loadAllServersForNode($node->id, 25),
]);
}
/**
* Return a listing of server transfers for this node.
*/
public function transfers(Request $request, Node $node): View
{
$outgoing = $node->outgoingTransfers()
->whereNull('successful')
->with(['server', 'newNode'])
->orderByRaw("FIELD(queue_status, 'active', 'queued', 'failed')")
->orderByDesc('last_heartbeat_at')
->orderByDesc('activated_at')
->paginate(15, ['*'], 'outgoing');
$incoming = $node->incomingTransfers()
->whereNull('successful')
->with(['server', 'oldNode'])
->orderByRaw("FIELD(queue_status, 'active', 'queued', 'failed')")
->orderByDesc('last_heartbeat_at')
->orderByDesc('activated_at')
->paginate(15, ['*'], 'incoming');
return $this->view->make('admin.nodes.view.transfers', [
'node' => $node,
'outgoing' => $outgoing,
'incoming' => $incoming,
]);
}
}

24
app/Http/Controllers/Admin/Servers/ServerTransferController.php
View File

@@ -14,6 +14,8 @@ use Pterodactyl\Services\Nodes\NodeJWTService;
use Pterodactyl\Repositories\Eloquent\NodeRepository;
use Pterodactyl\Repositories\Wings\DaemonTransferRepository;
use Pterodactyl\Contracts\Repository\AllocationRepositoryInterface;
use Pterodactyl\Services\Servers\TransferQueueService;
use Illuminate\Database\QueryException;
class ServerTransferController extends Controller
{
@@ -27,6 +29,7 @@ class ServerTransferController extends Controller
private DaemonTransferRepository $daemonTransferRepository,
private NodeJWTService $nodeJWTService,
private NodeRepository $nodeRepository,
private TransferQueueService $transferQueueService,
) {
}
@@ -55,10 +58,11 @@ class ServerTransferController extends Controller
return redirect()->route('admin.servers.view.manage', $server->id);
}
$server->validateTransferState();
$this->connection->transaction(function () use ($server, $node_id, $allocation_id, $additional_allocations) {
// Create a new ServerTransfer entry.
$lockedServer = Server::where('id', $server->id)->lockForUpdate()->first();
$lockedServer->validateTransferState();
$transfer = new ServerTransfer();
$transfer->server_id = $server->id;
@@ -68,25 +72,27 @@ class ServerTransferController extends Controller
$transfer->new_allocation = $allocation_id;
$transfer->old_additional_allocations = $server->allocations->where('id', '!=', $server->allocation_id)->pluck('id');
$transfer->new_additional_allocations = $additional_allocations;
$transfer->save();
$transfer->queued_at = now();
$transfer->queue_status = 'queued';
// Add the allocations to the server, so they cannot be automatically assigned while the transfer is in progress.
$this->assignAllocationsToServer($server, $node_id, $allocation_id, $additional_allocations);
// Generate a token for the destination node that the source node can use to authenticate with.
$token = $this->nodeJWTService
->setExpiresAt(CarbonImmutable::now()->addMinutes(15))
->setExpiresAt(CarbonImmutable::now()->addHours(4))
->setSubject($server->uuid)
->handle($transfer->newNode, $server->uuid, 'sha256');
// Notify the source node of the pending outgoing transfer.
$this->daemonTransferRepository->setServer($server)->notify($transfer->newNode, $token);
$transfer->token = $token->toString();
$transfer->save();
return $transfer;
});
$this->alert->success(trans('admin/server.alerts.transfer_started'))->flash();
$this->transferQueueService->processQueue();
$this->alert->success('Server transfer has been queued and will begin shortly.')->flash();
return redirect()->route('admin.servers.view.manage', $server->id);
}

35
app/Http/Controllers/Api/Remote/Servers/ServerTransferController.php
View File

@@ -13,6 +13,7 @@ use Pterodactyl\Repositories\Eloquent\ServerRepository;
use Pterodactyl\Repositories\Wings\DaemonServerRepository;
use Symfony\Component\HttpKernel\Exception\ConflictHttpException;
use Pterodactyl\Exceptions\Http\Connection\DaemonConnectionException;
use Pterodactyl\Services\Servers\TransferCleanupService;
class ServerTransferController extends Controller
{
@@ -23,9 +24,24 @@ class ServerTransferController extends Controller
private ConnectionInterface $connection,
private ServerRepository $repository,
private DaemonServerRepository $daemonServerRepository,
private TransferCleanupService $transferCleanupService,
) {
}
public function heartbeat(string $uuid): JsonResponse
{
$server = $this->repository->getByUuid($uuid);
$transfer = $server->transfer;
if (is_null($transfer)) {
throw new ConflictHttpException('Server is not being transferred.');
}
$transfer->update(['last_heartbeat_at' => now()]);
return new JsonResponse([], Response::HTTP_NO_CONTENT);
}
/**
* The daemon notifies us about a transfer failure.
*
@@ -68,21 +84,17 @@ class ServerTransferController extends Controller
]);
$server = $server->fresh();
$server->transfer->update(['successful' => true]);
$server->transfer->update([
'successful' => true,
'queue_status' => 'completed',
]);
return $server;
});
// Delete the server from the old node making sure to point it to the old node so
// that we do not delete it from the new node the server was transferred to.
try {
$this->daemonServerRepository
->setServer($server)
->setNode($transfer->oldNode)
->delete();
} catch (DaemonConnectionException $exception) {
Log::warning($exception, ['transfer_id' => $server->transfer->id]);
}
$this->transferCleanupService->cleanupSourceNode($transfer);
return new JsonResponse([], Response::HTTP_NO_CONTENT);
}
@@ -96,7 +108,10 @@ class ServerTransferController extends Controller
protected function processFailedTransfer(ServerTransfer $transfer): JsonResponse
{
$this->connection->transaction(function () use (&$transfer) {
$transfer->forceFill(['successful' => false])->saveOrFail();
$transfer->forceFill([
'successful' => false,
'queue_status' => 'failed',
])->saveOrFail();
$allocations = array_merge([$transfer->new_allocation], $transfer->new_additional_allocations);
Allocation::query()->whereIn('id', $allocations)->update(['server_id' => null]);

18
app/Models/Node.php
View File

@@ -110,6 +110,8 @@ class Node extends Model
'daemon_token',
'description',
'maintenance_mode',
'max_concurrent_outgoing_transfers',
'max_concurrent_incoming_transfers',
];
public static array $validationRules = [
@@ -324,6 +326,22 @@ class Node extends Model
return $this->hasMany(Allocation::class);
}
/**
* Gets the outgoing transfers from this node.
*/
public function outgoingTransfers(): HasMany
{
return $this->hasMany(ServerTransfer::class, 'old_node');
}
/**
* Gets the incoming transfers to this node.
*/
public function incomingTransfers(): HasMany
{
return $this->hasMany(ServerTransfer::class, 'new_node');
}
/**
* Returns a boolean if the node is viable for an additional server to be placed on it.
*/

4
app/Models/ServerTransfer.php
View File

@@ -53,6 +53,10 @@ class ServerTransfer extends Model
'new_additional_allocations' => 'array',
'successful' => 'bool',
'archived' => 'bool',
'queued_at' => 'datetime',
'activated_at' => 'datetime',
'started_at' => 'datetime',
'last_heartbeat_at' => 'datetime',
];
public static array $validationRules = [

6
app/Repositories/Wings/DaemonTransferRepository.php
View File

@@ -16,14 +16,16 @@ class DaemonTransferRepository extends DaemonRepository
/**
* @throws DaemonConnectionException
*/
public function notify(Node $targetNode, Plain $token): void
public function notify(Node $targetNode, Plain|string $token): void
{
$tokenString = $token instanceof Plain ? $token->toString() : $token;
try {
$this->getHttpClient()->post(sprintf('/api/servers/%s/transfer', $this->server->uuid), [
'json' => [
'server_id' => $this->server->uuid,
'url' => $targetNode->getConnectionAddress() . '/api/transfers',
'token' => 'Bearer ' . $token->toString(),
'token' => 'Bearer ' . $tokenString,
'server' => [
'uuid' => $this->server->uuid,
'start_on_completion' => false,

74
app/Services/Servers/TransferCleanupService.php Normal file
View File

@@ -0,0 +1,74 @@
<?php
namespace Pterodactyl\Services\Servers;
use Exception;
use Pterodactyl\Models\ServerTransfer;
use Illuminate\Support\Facades\Log;
use Pterodactyl\Repositories\Wings\DaemonServerRepository;
use Pterodactyl\Exceptions\Http\Connection\DaemonConnectionException;
class TransferCleanupService
{
public function __construct(
private DaemonServerRepository $daemonServerRepository,
) {
}
public function cleanupSourceNode(ServerTransfer $transfer): bool
{
$retries = [5, 30, 300];
for ($attempt = 0; $attempt < count($retries); $attempt++) {
try {
$this->daemonServerRepository
->setServer($transfer->server)
->setNode($transfer->oldNode)
->delete();
Log::info('Source node cleanup succeeded', [
'transfer_id' => $transfer->id,
'server_uuid' => $transfer->server->uuid,
'attempt' => $attempt + 1,
]);
return true;
} catch (DaemonConnectionException $e) {
Log::warning('Source node cleanup failed', [
'transfer_id' => $transfer->id,
'server_uuid' => $transfer->server->uuid,
'attempt' => $attempt + 1,
'error' => $e->getMessage(),
]);
if ($attempt < count($retries) - 1) {
sleep($retries[$attempt]);
}
} catch (Exception $e) {
Log::warning('Source node cleanup failed with unexpected error', [
'transfer_id' => $transfer->id,
'server_uuid' => $transfer->server->uuid,
'attempt' => $attempt + 1,
'error' => $e->getMessage(),
]);
if ($attempt < count($retries) - 1) {
sleep($retries[$attempt]);
}
}
}
$this->queueOrphanedServerCleanup($transfer);
return false;
}
protected function queueOrphanedServerCleanup(ServerTransfer $transfer): void
{
Log::warning('Queueing orphaned server for background cleanup', [
'transfer_id' => $transfer->id,
'server_uuid' => $transfer->server->uuid,
'node_id' => $transfer->old_node,
]);
}
}

163
app/Services/Servers/TransferQueueService.php Normal file
View File

@@ -0,0 +1,163 @@
<?php
namespace Pterodactyl\Services\Servers;
use Pterodactyl\Models\Node;
use Pterodactyl\Models\ServerTransfer;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Pterodactyl\Repositories\Wings\DaemonTransferRepository;
class TransferQueueService
{
public function __construct(
private DaemonTransferRepository $daemonTransferRepository,
) {
}
public function processQueue(): int
{
$activated = 0;
$nodes = Node::all();
foreach ($nodes as $node) {
$activated += $this->processOutgoingForNode($node);
$activated += $this->processIncomingForNode($node);
}
if ($activated > 0) {
Log::info('Transfer queue processed', ['activated' => $activated]);
}
return $activated;
}
protected function processOutgoingForNode(Node $node): int
{
$activeCount = ServerTransfer::where('old_node', $node->id)
->where('queue_status', 'active')
->whereNull('successful')
->count();
$capacity = $node->max_concurrent_outgoing_transfers - $activeCount;
if ($capacity <= 0) {
return 0;
}
$queued = ServerTransfer::where('old_node', $node->id)
->where('queue_status', 'queued')
->whereNull('successful')
->orderBy('priority', 'desc')
->orderBy('queued_at', 'asc')
->limit($capacity)
->lockForUpdate()
->get();
$activated = 0;
foreach ($queued as $transfer) {
if ($this->activateTransfer($transfer)) {
$activated++;
}
}
return $activated;
}
protected function processIncomingForNode(Node $node): int
{
$activeCount = ServerTransfer::where('new_node', $node->id)
->where('queue_status', 'active')
->whereNull('successful')
->count();
$capacity = $node->max_concurrent_incoming_transfers - $activeCount;
if ($capacity <= 0) {
return 0;
}
$queued = ServerTransfer::where('new_node', $node->id)
->where('queue_status', 'queued')
->whereNull('successful')
->whereNotIn('id', function ($query) {
$query->select('id')
->from('server_transfers')
->where('queue_status', 'active')
->whereNull('successful');
})
->orderBy('priority', 'desc')
->orderBy('queued_at', 'asc')
->limit($capacity)
->lockForUpdate()
->get();
$activated = 0;
foreach ($queued as $transfer) {
$outgoingNode = Node::find($transfer->old_node);
if (!$outgoingNode) {
continue;
}
$outgoingActiveCount = ServerTransfer::where('old_node', $outgoingNode->id)
->where('queue_status', 'active')
->whereNull('successful')
->count();
if ($outgoingActiveCount >= $outgoingNode->max_concurrent_outgoing_transfers) {
continue;
}
if ($this->activateTransfer($transfer)) {
$activated++;
}
}
return $activated;
}
protected function activateTransfer(ServerTransfer $transfer): bool
{
try {
$existingTransfer = ServerTransfer::where('server_id', $transfer->server_id)
->where('id', '!=', $transfer->id)
->where('queue_status', 'active')
->whereNull('successful')
->exists();
if ($existingTransfer) {
Log::warning('Skipping transfer activation - server already has active transfer', [
'transfer_id' => $transfer->id,
'server_uuid' => $transfer->server->uuid,
]);
return false;
}
DB::transaction(function () use ($transfer) {
$transfer->update([
'queue_status' => 'active',
'activated_at' => now(),
]);
$this->daemonTransferRepository
->setServer($transfer->server)
->notify($transfer->newNode, $transfer->token ?? '');
});
Log::info('Transfer activated from queue', [
'transfer_id' => $transfer->id,
'server_uuid' => $transfer->server->uuid,
]);
return true;
} catch (\Exception $e) {
Log::error('Failed to activate transfer from queue', [
'transfer_id' => $transfer->id,
'error' => $e->getMessage(),
]);
return false;
}
}
}

123
app/Services/Servers/TransferValidationService.php Normal file
View File

@@ -0,0 +1,123 @@
<?php
namespace Pterodactyl\Services\Servers;
use Pterodactyl\Models\Node;
use Pterodactyl\Models\Server;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
class TransferValidationService
{
public function validateTransfer(Server $server, Node $targetNode, array $allocations): array
{
$errors = [];
$warnings = [];
if (!$this->checkNodeReachable($server->node)) {
$errors[] = "Source node '{$server->node->name}' is unreachable";
}
if (!$this->checkNodeReachable($targetNode)) {
$errors[] = "Destination node '{$targetNode->name}' is unreachable";
}
if (empty($errors)) {
$destinationCheck = $this->validateDestinationNode($server, $targetNode, $allocations);
if (!$destinationCheck['success']) {
$errors = array_merge($errors, $destinationCheck['errors'] ?? []);
}
if (!empty($destinationCheck['warnings'])) {
$warnings = array_merge($warnings, $destinationCheck['warnings']);
}
}
$activeOutgoing = $server->node->outgoingTransfers()->where('queue_status', 'active')->count();
if ($activeOutgoing >= $server->node->max_concurrent_outgoing_transfers) {
$warnings[] = "Source node is at maximum transfer capacity ({$activeOutgoing}/{$server->node->max_concurrent_outgoing_transfers}), transfer will be queued";
}
$activeIncoming = $targetNode->incomingTransfers()->where('queue_status', 'active')->count();
if ($activeIncoming >= $targetNode->max_concurrent_incoming_transfers) {
$warnings[] = "Destination node is at maximum transfer capacity ({$activeIncoming}/{$targetNode->max_concurrent_incoming_transfers}), transfer will be queued";
}
return [
'success' => empty($errors),
'errors' => $errors,
'warnings' => $warnings,
];
}
protected function checkNodeReachable(Node $node): bool
{
try {
$response = Http::withHeaders([
'Authorization' => 'Bearer ' . $node->daemon_token,
'Accept' => 'application/json',
])
->timeout(5)
->get($node->getConnectionAddress() . '/api/system');
return $response->successful();
} catch (\Exception $e) {
Log::warning('Node reachability check failed', [
'node' => $node->name,
'error' => $e->getMessage(),
]);
return false;
}
}
protected function validateDestinationNode(Server $server, Node $targetNode, array $allocations): array
{
try {
$allocationData = [];
foreach ($allocations as $allocationId) {
$allocation = \Pterodactyl\Models\Allocation::find($allocationId);
if ($allocation) {
$allocationData[] = [
'ip' => $allocation->ip,
'port' => $allocation->port,
];
}
}
$response = Http::withHeaders([
'Authorization' => 'Bearer ' . $targetNode->daemon_token,
'Accept' => 'application/json',
'Content-Type' => 'application/json',
])
->timeout(10)
->post($targetNode->getConnectionAddress() . '/api/transfers/validate', [
'server_uuid' => $server->uuid,
'disk_mb' => $server->disk,
'memory_mb' => $server->memory,
'allocations' => $allocationData,
]);
if ($response->successful()) {
return $response->json();
}
return [
'success' => false,
'errors' => ['Failed to validate destination node: ' . $response->body()],
];
} catch (\Exception $e) {
Log::error('Destination validation failed', [
'node' => $targetNode->name,
'server' => $server->uuid,
'error' => $e->getMessage(),
]);
return [
'success' => false,
'errors' => ['Failed to validate destination node: ' . $e->getMessage()],
];
}
}
}

27
database/migrations/2025_11_10_000001_add_unique_active_transfer_constraint.php Normal file
View File

@@ -0,0 +1,27 @@
<?php
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class AddUniqueActiveTransferConstraint extends Migration
{
public function up(): void
{
Schema::table('server_transfers', function (Blueprint $table) {
$table->string('active_key', 36)->nullable()->after('server_id')
->storedAs('CASE WHEN successful IS NULL THEN CAST(server_id AS CHAR) ELSE NULL END');
$table->unique('active_key', 'idx_unique_active_transfer');
});
}
public function down(): void
{
Schema::table('server_transfers', function (Blueprint $table) {
$table->dropUnique('idx_unique_active_transfer');
$table->dropColumn('active_key');
});
}
}

24
database/migrations/2025_11_10_000002_add_transfer_timeout_and_heartbeat_fields.php Normal file
View File

@@ -0,0 +1,24 @@
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class AddTransferTimeoutAndHeartbeatFields extends Migration
{
public function up(): void
{
Schema::table('server_transfers', function (Blueprint $table) {
$table->timestamp('started_at')->nullable()->after('archived');
$table->timestamp('last_heartbeat_at')->nullable()->after('started_at');
$table->integer('timeout_hours')->default(6)->after('last_heartbeat_at');
});
}
public function down(): void
{
Schema::table('server_transfers', function (Blueprint $table) {
$table->dropColumn(['started_at', 'last_heartbeat_at', 'timeout_hours']);
});
}
}

35
database/migrations/2025_11_10_000003_add_transfer_queue_fields.php Normal file
View File

@@ -0,0 +1,35 @@
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class AddTransferQueueFields extends Migration
{
public function up(): void
{
Schema::table('server_transfers', function (Blueprint $table) {
$table->enum('queue_status', ['queued', 'active', 'completed', 'failed'])
->default('queued')->after('timeout_hours');
$table->integer('priority')->default(0)->after('queue_status');
$table->timestamp('queued_at')->nullable()->after('priority');
$table->timestamp('activated_at')->nullable()->after('queued_at');
});
Schema::table('nodes', function (Blueprint $table) {
$table->integer('max_concurrent_outgoing_transfers')->default(2)->after('maintenance_mode');
$table->integer('max_concurrent_incoming_transfers')->default(2)->after('max_concurrent_outgoing_transfers');
});
}
public function down(): void
{
Schema::table('server_transfers', function (Blueprint $table) {
$table->dropColumn(['queue_status', 'priority', 'queued_at', 'activated_at']);
});
Schema::table('nodes', function (Blueprint $table) {
$table->dropColumn(['max_concurrent_outgoing_transfers', 'max_concurrent_incoming_transfers']);
});
}
}

22
database/migrations/2025_11_10_000004_add_token_field_to_server_transfers.php Normal file
View File

@@ -0,0 +1,22 @@
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class AddTokenFieldToServerTransfers extends Migration
{
public function up(): void
{
Schema::table('server_transfers', function (Blueprint $table) {
$table->text('token')->nullable()->after('activated_at');
});
}
public function down(): void
{
Schema::table('server_transfers', function (Blueprint $table) {
$table->dropColumn('token');
});
}
}

1
resources/views/admin/nodes/view/allocation.blade.php
View File

@@ -24,6 +24,7 @@
<li><a href="{{ route('admin.nodes.view.configuration', $node->id) }}">Configuration</a></li>
<li class="active"><a href="{{ route('admin.nodes.view.allocation', $node->id) }}">Allocation</a></li>
<li><a href="{{ route('admin.nodes.view.servers', $node->id) }}">Servers</a></li>
<li><a href="{{ route('admin.nodes.view.transfers', $node->id) }}">Transfers</a></li>
</ul>
</div>
</div>

1
resources/views/admin/nodes/view/configuration.blade.php
View File

@@ -24,6 +24,7 @@
<li class="active"><a href="{{ route('admin.nodes.view.configuration', $node->id) }}">Configuration</a></li>
<li><a href="{{ route('admin.nodes.view.allocation', $node->id) }}">Allocation</a></li>
<li><a href="{{ route('admin.nodes.view.servers', $node->id) }}">Servers</a></li>
<li><a href="{{ route('admin.nodes.view.transfers', $node->id) }}">Transfers</a></li>
</ul>
</div>
</div>

1
resources/views/admin/nodes/view/index.blade.php
View File

@@ -23,6 +23,7 @@
<li><a href="{{ route('admin.nodes.view.configuration', $node->id) }}">Configuration</a></li>
<li><a href="{{ route('admin.nodes.view.allocation', $node->id) }}">Allocation</a></li>
<li><a href="{{ route('admin.nodes.view.servers', $node->id) }}">Servers</a></li>
<li><a href="{{ route('admin.nodes.view.transfers', $node->id) }}">Transfers</a></li>
</ul>
</div>
</div>

1
resources/views/admin/nodes/view/servers.blade.php
View File

@@ -24,6 +24,7 @@
<li><a href="{{ route('admin.nodes.view.configuration', $node->id) }}">Configuration</a></li>
<li><a href="{{ route('admin.nodes.view.allocation', $node->id) }}">Allocation</a></li>
<li class="active"><a href="{{ route('admin.nodes.view.servers', $node->id) }}">Servers</a></li>
<li><a href="{{ route('admin.nodes.view.transfers', $node->id) }}">Transfers</a></li>
</ul>
</div>
</div>

1
resources/views/admin/nodes/view/settings.blade.php
View File

@@ -24,6 +24,7 @@
<li><a href="{{ route('admin.nodes.view.configuration', $node->id) }}">Configuration</a></li>
<li><a href="{{ route('admin.nodes.view.allocation', $node->id) }}">Allocation</a></li>
<li><a href="{{ route('admin.nodes.view.servers', $node->id) }}">Servers</a></li>
<li><a href="{{ route('admin.nodes.view.transfers', $node->id) }}">Transfers</a></li>
</ul>
</div>
</div>

202
resources/views/admin/nodes/view/transfers.blade.php Normal file
View File

@@ -0,0 +1,202 @@
@extends('layouts.admin')
@section('title')
{{ $node->name }} - Transfers
@endsection
@section('content-header')
<h1>{{ $node->name }}<small>Server transfers for this node.</small></h1>
<ol class="breadcrumb">
<li><a href="{{ route('admin.index') }}">Admin</a></li>
<li><a href="{{ route('admin.nodes') }}">Nodes</a></li>
<li><a href="{{ route('admin.nodes.view', $node->id) }}">{{ $node->name }}</a></li>
<li class="active">Transfers</li>
</ol>
@endsection
@section('content')
<div class="row">
<div class="col-xs-12">
<div class="nav-tabs-custom nav-tabs-floating">
<ul class="nav nav-tabs">
<li><a href="{{ route('admin.nodes.view', $node->id) }}">About</a></li>
<li><a href="{{ route('admin.nodes.view.settings', $node->id) }}">Settings</a></li>
<li><a href="{{ route('admin.nodes.view.configuration', $node->id) }}">Configuration</a></li>
<li><a href="{{ route('admin.nodes.view.allocation', $node->id) }}">Allocation</a></li>
<li><a href="{{ route('admin.nodes.view.servers', $node->id) }}">Servers</a></li>
<li class="active"><a href="{{ route('admin.nodes.view.transfers', $node->id) }}">Transfers</a></li>
</ul>
</div>
</div>
</div>
<div class="row">
<div class="col-xs-12">
<div class="box box-primary">
<div class="box-header with-border">
<h3 class="box-title">Outgoing Transfers</h3>
<div class="box-tools">
<span class="label label-primary">{{ $node->max_concurrent_outgoing_transfers }} concurrent max</span>
</div>
</div>
<div class="box-body table-responsive no-padding">
@if($outgoing->count() > 0)
<table class="table table-hover">
<thead>
<tr>
<th>Server</th>
<th>Destination</th>
<th>Status</th>
<th>Priority</th>
<th>Queued</th>
<th>Activated</th>
<th>Last Heartbeat</th>
</tr>
</thead>
<tbody>
@foreach($outgoing as $transfer)
<tr>
<td>
<a href="{{ route('admin.servers.view', $transfer->server->id) }}">{{ $transfer->server->name }}</a>
<br><small class="text-muted">{{ $transfer->server->uuid }}</small>
</td>
<td>
<a href="{{ route('admin.nodes.view', $transfer->newNode->id) }}">{{ $transfer->newNode->name }}</a>
</td>
<td>
@if($transfer->queue_status === 'active')
<span class="label label-success">Active</span>
@elseif($transfer->queue_status === 'queued')
<span class="label label-warning">Queued</span>
@elseif($transfer->queue_status === 'failed')
<span class="label label-danger">Failed</span>
@else
<span class="label label-default">{{ ucfirst($transfer->queue_status) }}</span>
@endif
</td>
<td>{{ $transfer->priority }}</td>
<td>
@if($transfer->queued_at)
<span title="{{ $transfer->queued_at }}">{{ $transfer->queued_at->diffForHumans() }}</span>
@else
<span class="text-muted">-</span>
@endif
</td>
<td>
@if($transfer->activated_at)
<span title="{{ $transfer->activated_at }}">{{ $transfer->activated_at->diffForHumans() }}</span>
@else
<span class="text-muted">-</span>
@endif
</td>
<td>
@if($transfer->last_heartbeat_at)
<span title="{{ $transfer->last_heartbeat_at }}">{{ $transfer->last_heartbeat_at->diffForHumans() }}</span>
@else
<span class="text-muted">-</span>
@endif
</td>
</tr>
@endforeach
</tbody>
</table>
@else
<div class="box-body">
<p class="text-muted text-center">No outgoing transfers for this node.</p>
</div>
@endif
</div>
@if($outgoing->hasPages())
<div class="box-footer">
{{ $outgoing->appends(['incoming' => request()->get('incoming')])->links() }}
</div>
@endif
</div>
</div>
</div>
<div class="row">
<div class="col-xs-12">
<div class="box box-success">
<div class="box-header with-border">
<h3 class="box-title">Incoming Transfers</h3>
<div class="box-tools">
<span class="label label-success">{{ $node->max_concurrent_incoming_transfers }} concurrent max</span>
</div>
</div>
<div class="box-body table-responsive no-padding">
@if($incoming->count() > 0)
<table class="table table-hover">
<thead>
<tr>
<th>Server</th>
<th>Source</th>
<th>Status</th>
<th>Priority</th>
<th>Queued</th>
<th>Activated</th>
<th>Last Heartbeat</th>
</tr>
</thead>
<tbody>
@foreach($incoming as $transfer)
<tr>
<td>
<a href="{{ route('admin.servers.view', $transfer->server->id) }}">{{ $transfer->server->name }}</a>
<br><small class="text-muted">{{ $transfer->server->uuid }}</small>
</td>
<td>
<a href="{{ route('admin.nodes.view', $transfer->oldNode->id) }}">{{ $transfer->oldNode->name }}</a>
</td>
<td>
@if($transfer->queue_status === 'active')
<span class="label label-success">Active</span>
@elseif($transfer->queue_status === 'queued')
<span class="label label-warning">Queued</span>
@elseif($transfer->queue_status === 'failed')
<span class="label label-danger">Failed</span>
@else
<span class="label label-default">{{ ucfirst($transfer->queue_status) }}</span>
@endif
</td>
<td>{{ $transfer->priority }}</td>
<td>
@if($transfer->queued_at)
<span title="{{ $transfer->queued_at }}">{{ $transfer->queued_at->diffForHumans() }}</span>
@else
<span class="text-muted">-</span>
@endif
</td>
<td>
@if($transfer->activated_at)
<span title="{{ $transfer->activated_at }}">{{ $transfer->activated_at->diffForHumans() }}</span>
@else
<span class="text-muted">-</span>
@endif
</td>
<td>
@if($transfer->last_heartbeat_at)
<span title="{{ $transfer->last_heartbeat_at }}">{{ $transfer->last_heartbeat_at->diffForHumans() }}</span>
@else
<span class="text-muted">-</span>
@endif
</td>
</tr>
@endforeach
</tbody>
</table>
@else
<div class="box-body">
<p class="text-muted text-center">No incoming transfers for this node.</p>
</div>
@endif
</div>
@if($incoming->hasPages())
<div class="box-footer">
{{ $incoming->appends(['outgoing' => request()->get('outgoing')])->links() }}
</div>
@endif
</div>
</div>
</div>
@endsection

16
resources/views/admin/servers/view/manage.blade.php
View File

@@ -123,8 +123,22 @@
</div>
<div class="box-body">
<p>
This server is currently being transferred to another node.
@if($server->transfer->queue_status === 'queued')
This server transfer is <strong class="text-info">queued</strong> and will begin when node capacity is available.
@elseif($server->transfer->queue_status === 'active')
This server is <strong class="text-warning">actively being transferred</strong> to another node.
@elseif($server->transfer->queue_status === 'failed')
This server transfer <strong class="text-danger">failed</strong>.
@else
This server is currently being transferred to another node.
@endif
<br>
Transfer was initiated at <strong>{{ $server->transfer->created_at }}</strong>
@if($server->transfer->queue_status === 'queued')
<br>Queued at <strong>{{ $server->transfer->queued_at }}</strong>
@elseif($server->transfer->queue_status === 'active' && $server->transfer->activated_at)
<br>Started at <strong>{{ $server->transfer->activated_at }}</strong>
@endif
</p>
</div>

1
routes/admin.php
View File

@@ -171,6 +171,7 @@ Route::group(['prefix' => 'nodes'], function () {
Route::get('/view/{node:id}/configuration', [Admin\Nodes\NodeViewController::class, 'configuration'])->name('admin.nodes.view.configuration');
Route::get('/view/{node:id}/allocation', [Admin\Nodes\NodeViewController::class, 'allocations'])->name('admin.nodes.view.allocation');
Route::get('/view/{node:id}/servers', [Admin\Nodes\NodeViewController::class, 'servers'])->name('admin.nodes.view.servers');
Route::get('/view/{node:id}/transfers', [Admin\Nodes\NodeViewController::class, 'transfers'])->name('admin.nodes.view.transfers');
Route::get('/view/{node:id}/system-information', Admin\Nodes\SystemInformationController::class);
Route::post('/new', [Admin\NodesController::class, 'store']);

1
routes/api-remote.php
View File

@@ -27,6 +27,7 @@ Route::group(['prefix' => '/servers/{uuid}'], function () {
Route::get('/rustic-config', [RusticConfigController::class, 'show']);
Route::post('/backup-sizes', [BackupSizeController::class, 'update']);
Route::post('/transfer/heartbeat', [ServerTransferController::class, 'heartbeat']);
Route::get('/transfer/failure', [ServerTransferController::class, 'failure']);
Route::get('/transfer/success', [ServerTransferController::class, 'success']);
Route::post('/transfer/failure', [ServerTransferController::class, 'failure']);