Intermittently all items in VW fail to load with 500 Internal Server Error and 'Error loading ciphers: DatabaseError(Unknown, "cache entry already complete")' #1331

New Issue

MrUnknownDE · 2026-04-06T01:45:56+02:00

MrUnknownDE commented

2026-04-06 01:45:56 +02:00

Originally created by @ccben87 on 12/4/2023

Subject of the issue

Intermittently all items in Vault do not display and browser receives 500 Internal Server Error. Logs indicate "[panic][ERROR] thread 'rocket-worker-thread' panicked at 'Error loading ciphers: DatabaseError(Unknown, "cache entry already complete")'"

Logging out and in a few times eventually causes passwords to load and once loaded it continues to work. Multiple users have had this issue.

Deployment environment

Your environment (Generated via diagnostics page)

Vaultwarden version: v1.30.0-48836501
Web-vault version: v2023.10.0
OS/Arch: linux/x86_64
Running within Docker: false (Base: Not applicable)
Environment settings overridden: false
Uses a reverse proxy: true
IP Header check: true (X-Real-IP)
Internet access: true
Internet access via a proxy: true
DNS Check: true
Browser/Server Time Check: true
Server/NTP Time Check: true
Domain Configuration Check: true
HTTPS Check: true
Database type: PostgreSQL
Database version: PostgreSQL 16.0 (Debian 16.0-1.pgdg110+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
Clients used:
Reverse proxy and version:
Other relevant information:

Config (Generated via diagnostics page)

Show Running Config

Environment settings which are overridden:

{
  "_duo_akey": null,
  "_enable_duo": true,
  "_enable_email_2fa": true,
  "_enable_smtp": true,
  "_enable_yubico": true,
  "_icon_service_csp": "",
  "_icon_service_url": "",
  "_ip_header_enabled": true,
  "_smtp_img_src": "cid:",
  "admin_ratelimit_max_burst": 3,
  "admin_ratelimit_seconds": 300,
  "admin_session_lifetime": 20,
  "admin_token": "***",
  "allowed_iframe_ancestors": "",
  "attachments_folder": "data/attachments",
  "auth_request_purge_schedule": "30 * * * * *",
  "authenticator_disable_time_drift": false,
  "data_folder": "data",
  "database_conn_init": "",
  "database_max_conns": 10,
  "database_timeout": 30,
  "database_url": "**********://********************************************************************************************************************************************",
  "db_connection_retries": 15,
  "disable_2fa_remember": true,
  "disable_admin_token": false,
  "disable_icon_download": false,
  "domain": "*****://**********************",
  "domain_origin": "*****://**********************",
  "domain_path": "",
  "domain_set": true,
  "duo_host": null,
  "duo_ikey": null,
  "duo_skey": null,
  "email_attempts_limit": 3,
  "email_change_allowed": true,
  "email_expiration_time": 600,
  "email_token_size": 6,
  "emergency_access_allowed": true,
  "emergency_notification_reminder_schedule": "0 3 * * * *",
  "emergency_request_timeout_schedule": "0 7 * * * *",
  "enable_db_wal": true,
  "event_cleanup_schedule": "0 10 0 * * *",
  "events_days_retain": null,
  "extended_logging": true,
  "helo_name": null,
  "hibp_api_key": null,
  "icon_blacklist_non_global_ips": true,
  "icon_blacklist_regex": null,
  "icon_cache_folder": "data/icon_cache",
  "icon_cache_negttl": 259200,
  "icon_cache_ttl": 2592000,
  "icon_download_timeout": 10,
  "icon_redirect_code": 302,
  "icon_service": "internal",
  "incomplete_2fa_schedule": "30 * * * * *",
  "incomplete_2fa_time_limit": 3,
  "invitation_expiration_hours": 120,
  "invitation_org_name": "Vaultwarden",
  "invitations_allowed": true,
  "ip_header": "X-Real-IP",
  "job_poll_interval_ms": 30000,
  "log_file": null,
  "log_level": "Info",
  "log_timestamp_format": "%Y-%m-%d %H:%M:%S.%3f",
  "login_ratelimit_max_burst": 10,
  "login_ratelimit_seconds": 60,
  "org_attachment_limit": 0,
  "org_creation_users": "***********************",
  "org_events_enabled": true,
  "org_groups_enabled": true,
  "password_hints_allowed": true,
  "password_iterations": 600000,
  "push_enabled": false,
  "push_installation_id": "***",
  "push_installation_key": "***",
  "push_relay_uri": "https://push.bitwarden.com",
  "reload_templates": false,
  "require_device_email": false,
  "rsa_key_filename": "data/rsa_key",
  "send_purge_schedule": "0 5 * * * *",
  "sendmail_command": null,
  "sends_allowed": true,
  "sends_folder": "data/sends",
  "show_password_hint": false,
  "signups_allowed": false,
  "signups_domains_whitelist": "",
  "signups_verify": true,
  "signups_verify_resend_limit": 6,
  "signups_verify_resend_time": 3600,
  "smtp_accept_invalid_certs": false,
  "smtp_accept_invalid_hostnames": false,
  "smtp_auth_mechanism": null,
  "smtp_debug": false,
  "smtp_embed_images": true,
  "smtp_explicit_tls": null,
  "smtp_from": "****************************",
  "smtp_from_name": "Vaultwarden",
  "smtp_host": "***************************************",
  "smtp_password": "***",
  "smtp_port": 465,
  "smtp_security": "force_tls",
  "smtp_ssl": null,
  "smtp_timeout": 15,
  "smtp_username": "********************",
  "templates_folder": "data/templates",
  "tmp_folder": "data/tmp",
  "trash_auto_delete_days": null,
  "trash_purge_schedule": "0 5 0 * * *",
  "use_sendmail": false,
  "use_syslog": false,
  "user_attachment_limit": 0,
  "web_vault_enabled": true,
  "web_vault_folder": "web-vault/",
  "websocket_address": "0.0.0.0",
  "websocket_enabled": false,
  "websocket_port": 3012,
  "yubico_client_id": null,
  "yubico_secret_key": null,
  "yubico_server": null
}

Install method:
Running Docker testing-alpine image inside Kubernetes environment (single replica of Vaultwarden) and connecting to a cluster of 3 Postgres servers (1 primary, 2 replicas) with asynchronous replication.
Clients used:
Web Vault
Reverse proxy and version:
Kong Gateway proxying to NGINX
MySQL/MariaDB or PostgreSQL version:
Postgres 16.0
Other relevant details:

Steps to reproduce

Unsure how to reproduce. This has just started out of nowhere after using Vaultwarden for some time. Possibly somehow DB has become corrupted?

Expected behaviour

Web Vault should always load passwords. DB errors are occurring.

Actual behaviour

Sometimes when signing into Web Vault the passwords don't display and the browser receives a 500 Internal Server Error (logs indicate a database error). Otherwise clicking about the various pages works but no passwords are listed. After signing in and out a few times, the experience returns to normal.

Troubleshooting data

<!-- Share any log files, screenshots, or other relevant troubleshooting data -->
[2023-12-04 03:57:11.257][request][INFO] GET /api/sync?excludeDomains=true
[2023-12-04 03:57:11.269][panic][ERROR] thread 'rocket-worker-thread' panicked at 'Error loading ciphers: DatabaseError(Unknown, "cache entry already complete")': src/db/models/cipher.rs:638
   0: vaultwarden::init_logging::{{closure}}
   1: std::panicking::rust_panic_with_hook
   2: std::panicking::begin_panic_handler::{{closure}}
   3: std::sys_common::backtrace::__rust_end_short_backtrace
   4: rust_begin_unwind
   5: core::panicking::panic_fmt
   6: core::result::unwrap_failed
   7: vaultwarden::db::models::cipher::Cipher::find_by_user::{{closure}}::{{closure}}
   8: tokio::runtime::scheduler::multi_thread::worker::block_in_place
   9: vaultwarden::db::models::cipher::Cipher::find_by_user_visible::{{closure}}
  10: vaultwarden::api::core::ciphers::sync::{{closure}}
  11: vaultwarden::api::core::ciphers::sync::into_info::monomorphized_function::{{closure}}
  12: rocket::server::<impl rocket::rocket::Rocket<rocket::phase::Orbit>>::route::{{closure}}
  13: rocket::server::hyper_service_fn::{{closure}}::{{closure}}
  14: tokio::runtime::task::core::Core<T,S>::poll
  15: tokio::runtime::task::harness::Harness<T,S>::poll
  16: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
  17: tokio::runtime::scheduler::multi_thread::worker::run
  18: tokio::runtime::task::core::Core<T,S>::poll
  19: tokio::runtime::task::harness::Harness<T,S>::poll
  20: std::sys_common::backtrace::__rust_begin_short_backtrace
  21: core::ops::function::FnOnce::call_once{{vtable.shim}}
  22: std::sys::unix::thread::Thread::new::thread_start

[2023-12-04 03:57:11.275][response][INFO] (sync) GET /api/sync?<data..> => 500 Internal Server Error

*Originally created by @ccben87 on 12/4/2023* ### Subject of the issue Intermittently all items in Vault do not display and browser receives 500 Internal Server Error. Logs indicate "[panic][ERROR] thread 'rocket-worker-thread' panicked at 'Error loading ciphers: DatabaseError(Unknown, "cache entry already complete")'" Logging out and in a few times eventually causes passwords to load and once loaded it continues to work. Multiple users have had this issue. ### Deployment environment ### Your environment (Generated via diagnostics page) * Vaultwarden version: v1.30.0-48836501 * Web-vault version: v2023.10.0 * OS/Arch: linux/x86_64 * Running within Docker: false (Base: Not applicable) * Environment settings overridden: false * Uses a reverse proxy: true * IP Header check: true (X-Real-IP) * Internet access: true * Internet access via a proxy: true * DNS Check: true * Browser/Server Time Check: true * Server/NTP Time Check: true * Domain Configuration Check: true * HTTPS Check: true * Database type: PostgreSQL * Database version: PostgreSQL 16.0 (Debian 16.0-1.pgdg110+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit * Clients used: * Reverse proxy and version: * Other relevant information: ### Config (Generated via diagnostics page) <details><summary>Show Running Config</summary> **Environment settings which are overridden:** ```json { "_duo_akey": null, "_enable_duo": true, "_enable_email_2fa": true, "_enable_smtp": true, "_enable_yubico": true, "_icon_service_csp": "", "_icon_service_url": "", "_ip_header_enabled": true, "_smtp_img_src": "cid:", "admin_ratelimit_max_burst": 3, "admin_ratelimit_seconds": 300, "admin_session_lifetime": 20, "admin_token": "***", "allowed_iframe_ancestors": "", "attachments_folder": "data/attachments", "auth_request_purge_schedule": "30 * * * * *", "authenticator_disable_time_drift": false, "data_folder": "data", "database_conn_init": "", "database_max_conns": 10, "database_timeout": 30, "database_url": "**********://********************************************************************************************************************************************", "db_connection_retries": 15, "disable_2fa_remember": true, "disable_admin_token": false, "disable_icon_download": false, "domain": "*****://**********************", "domain_origin": "*****://**********************", "domain_path": "", "domain_set": true, "duo_host": null, "duo_ikey": null, "duo_skey": null, "email_attempts_limit": 3, "email_change_allowed": true, "email_expiration_time": 600, "email_token_size": 6, "emergency_access_allowed": true, "emergency_notification_reminder_schedule": "0 3 * * * *", "emergency_request_timeout_schedule": "0 7 * * * *", "enable_db_wal": true, "event_cleanup_schedule": "0 10 0 * * *", "events_days_retain": null, "extended_logging": true, "helo_name": null, "hibp_api_key": null, "icon_blacklist_non_global_ips": true, "icon_blacklist_regex": null, "icon_cache_folder": "data/icon_cache", "icon_cache_negttl": 259200, "icon_cache_ttl": 2592000, "icon_download_timeout": 10, "icon_redirect_code": 302, "icon_service": "internal", "incomplete_2fa_schedule": "30 * * * * *", "incomplete_2fa_time_limit": 3, "invitation_expiration_hours": 120, "invitation_org_name": "Vaultwarden", "invitations_allowed": true, "ip_header": "X-Real-IP", "job_poll_interval_ms": 30000, "log_file": null, "log_level": "Info", "log_timestamp_format": "%Y-%m-%d %H:%M:%S.%3f", "login_ratelimit_max_burst": 10, "login_ratelimit_seconds": 60, "org_attachment_limit": 0, "org_creation_users": "***********************", "org_events_enabled": true, "org_groups_enabled": true, "password_hints_allowed": true, "password_iterations": 600000, "push_enabled": false, "push_installation_id": "***", "push_installation_key": "***", "push_relay_uri": "https://push.bitwarden.com", "reload_templates": false, "require_device_email": false, "rsa_key_filename": "data/rsa_key", "send_purge_schedule": "0 5 * * * *", "sendmail_command": null, "sends_allowed": true, "sends_folder": "data/sends", "show_password_hint": false, "signups_allowed": false, "signups_domains_whitelist": "", "signups_verify": true, "signups_verify_resend_limit": 6, "signups_verify_resend_time": 3600, "smtp_accept_invalid_certs": false, "smtp_accept_invalid_hostnames": false, "smtp_auth_mechanism": null, "smtp_debug": false, "smtp_embed_images": true, "smtp_explicit_tls": null, "smtp_from": "****************************", "smtp_from_name": "Vaultwarden", "smtp_host": "***************************************", "smtp_password": "***", "smtp_port": 465, "smtp_security": "force_tls", "smtp_ssl": null, "smtp_timeout": 15, "smtp_username": "********************", "templates_folder": "data/templates", "tmp_folder": "data/tmp", "trash_auto_delete_days": null, "trash_purge_schedule": "0 5 0 * * *", "use_sendmail": false, "use_syslog": false, "user_attachment_limit": 0, "web_vault_enabled": true, "web_vault_folder": "web-vault/", "websocket_address": "0.0.0.0", "websocket_enabled": false, "websocket_port": 3012, "yubico_client_id": null, "yubico_secret_key": null, "yubico_server": null } ``` </details>  * Install method: Running Docker testing-alpine image inside Kubernetes environment (single replica of Vaultwarden) and connecting to a cluster of 3 Postgres servers (1 primary, 2 replicas) with asynchronous replication. * Clients used:  Web Vault * Reverse proxy and version:  Kong Gateway proxying to NGINX * MySQL/MariaDB or PostgreSQL version:  Postgres 16.0 * Other relevant details: ### Steps to reproduce  Unsure how to reproduce. This has just started out of nowhere after using Vaultwarden for some time. Possibly somehow DB has become corrupted? ### Expected behaviour  Web Vault should always load passwords. DB errors are occurring. ### Actual behaviour  Sometimes when signing into Web Vault the passwords don't display and the browser receives a 500 Internal Server Error (logs indicate a database error). Otherwise clicking about the various pages works but no passwords are listed. After signing in and out a few times, the experience returns to normal. ### Troubleshooting data ```  [2023-12-04 03:57:11.257][request][INFO] GET /api/sync?excludeDomains=true [2023-12-04 03:57:11.269][panic][ERROR] thread 'rocket-worker-thread' panicked at 'Error loading ciphers: DatabaseError(Unknown, "cache entry already complete")': src/db/models/cipher.rs:638 0: vaultwarden::init_logging::{{closure}} 1: std::panicking::rust_panic_with_hook 2: std::panicking::begin_panic_handler::{{closure}} 3: std::sys_common::backtrace::__rust_end_short_backtrace 4: rust_begin_unwind 5: core::panicking::panic_fmt 6: core::result::unwrap_failed 7: vaultwarden::db::models::cipher::Cipher::find_by_user::{{closure}}::{{closure}} 8: tokio::runtime::scheduler::multi_thread::worker::block_in_place 9: vaultwarden::db::models::cipher::Cipher::find_by_user_visible::{{closure}} 10: vaultwarden::api::core::ciphers::sync::{{closure}} 11: vaultwarden::api::core::ciphers::sync::into_info::monomorphized_function::{{closure}} 12: rocket::server::<impl rocket::rocket::Rocket<rocket::phase::Orbit>>::route::{{closure}} 13: rocket::server::hyper_service_fn::{{closure}}::{{closure}} 14: tokio::runtime::task::core::Core<T,S>::poll 15: tokio::runtime::task::harness::Harness<T,S>::poll 16: tokio::runtime::scheduler::multi_thread::worker::Context::run_task 17: tokio::runtime::scheduler::multi_thread::worker::run 18: tokio::runtime::task::core::Core<T,S>::poll 19: tokio::runtime::task::harness::Harness<T,S>::poll 20: std::sys_common::backtrace::__rust_begin_short_backtrace 21: core::ops::function::FnOnce::call_once{{vtable.shim}} 22: std::sys::unix::thread::Thread::new::thread_start [2023-12-04 03:57:11.275][response][INFO] (sync) GET /api/sync?<data..> => 500 Internal Server Error ```

MrUnknownDE closed this issue

2026-04-06 01:45:56 +02:00

Sign in to join this conversation.

No Label

1 Participants

Notifications

Due Date

No due date set.

Dependencies

No dependencies set.

Reference: github/vaultwarden#1331