From 8ccc8af7101a9c605075b4513052b25b9f7dd033 Mon Sep 17 00:00:00 2001 From: Kyler Li <46853114+loukylor@users.noreply.github.com> Date: Mon, 28 Apr 2025 03:17:12 -0700 Subject: [PATCH] Add database merging feature (#906) * add initial implementation of dbmerger merges of user tables that have overlapping data is still unimplemented. otherwise, merging of all other tables has been implemented. * add handling for when user feed tables overlap * fix sqlite dll path in dbmerger build * fix accidentally inserting rows with nonunique pks * fix merger not handling missing columns well * fix sort not reassigning pks * sync merger with master * woopsies * w-woops again * reset database version in config to let vrcx generate missing fields * move to net 9 and move build folder to new build folder * remove unneeded build configurations * support avatar time in the merge * csproj pains --------- Co-authored-by: Natsumi --- DBMerger/Config.cs | 10 + DBMerger/DBMerger.csproj | 50 +++ DBMerger/Merger.cs | 629 +++++++++++++++++++++++++++++++++++ DBMerger/Program.cs | 209 ++++++++++++ DBMerger/SqliteExtensions.cs | 77 +++++ Dotnet/VRCX-Cef.csproj | 9 +- Dotnet/VRCX-Electron.csproj | 8 +- VRCX.sln | 6 + 8 files changed, 993 insertions(+), 5 deletions(-) create mode 100644 DBMerger/Config.cs create mode 100644 DBMerger/DBMerger.csproj create mode 100644 DBMerger/Merger.cs create mode 100644 DBMerger/Program.cs create mode 100644 DBMerger/SqliteExtensions.cs diff --git a/DBMerger/Config.cs b/DBMerger/Config.cs new file mode 100644 index 00000000..ef236790 --- /dev/null +++ b/DBMerger/Config.cs @@ -0,0 +1,10 @@ +namespace DBMerger +{ + public class Config(string newDBPath, string oldDBPath, bool debug, bool importConfig) + { + public string NewDBPath { get; } = newDBPath; + public string OldDBPath { get; } = oldDBPath; + public bool Debug { get; } = debug; + public bool ImportConfig { get; } = importConfig; + } +} diff --git a/DBMerger/DBMerger.csproj b/DBMerger/DBMerger.csproj new file mode 100644 index 00000000..18a9161b --- /dev/null +++ b/DBMerger/DBMerger.csproj @@ -0,0 +1,50 @@ + + + ..\build\Cef\ + + + Exe + net9 + x64 + 0 + 1.0.0.%2a + DBMerger + DBMerger + vrcx-team, loukylor, pypy, natsumi + false + false + DBMerger + DBMerger - Merges VRCX sqlite databases + https://github.com/vrcx-team/VRCX + en + en-US;en + true + + false + + + + full + + + pdbonly + + + + win-x64 + false + + + + true + true + false + false + + + + + + + + diff --git a/DBMerger/Merger.cs b/DBMerger/Merger.cs new file mode 100644 index 00000000..a3ded0ca --- /dev/null +++ b/DBMerger/Merger.cs @@ -0,0 +1,629 @@ +using NLog; +using NLog.Fluent; +using SQLite; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; + +namespace DBMerger +{ + public partial class Merger(SQLiteConnection dbConn, string oldDBName, string newDBName, Config config) + { + // C#'s iso date string has millionths of a second but the db stores + // dates with only thousands of a second, so define our own format + private const string JSDateTimeFormat = "yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fffzzz"; + + private static readonly Logger logger = LogManager.GetCurrentClassLogger(); + private static readonly Regex userIDRegex = UserIDRegex(); + + // This list of table names will be slowly emptied by specific + // handlers until only unrecognied tables are left + private List unMergedTables; + + public void Merge() + { + logger.Info("Starting merge process..."); + + logger.Debug("Creating transaction for databases"); + dbConn.BeginTransaction(); + try + { + MergeInternal(); + } + catch + { + logger.Warn("Encoutered error! Rolling back changes to databases"); + dbConn.Rollback(); + + throw; + } + + logger.Debug("Committing changes to database"); + dbConn.Commit(); + + logger.Info("Optimizing database size..."); + dbConn.Execute("VACUUM new_db;"); + logger.Info("Merge completed without any major issues!"); + } + + private void MergeInternal() + { + unMergedTables = dbConn.QueryScalars($"SELECT name FROM {oldDBName}.sqlite_schema WHERE type='table';"); + + // Holds sensitive information. Burn it with fire so no sensitive + // data gets leaked + unMergedTables.Remove("cookies"); + + // Get any tables in the old db that arent in the new db + logger.Info("Creating tables not present on new database that are present on old database..."); + var newDBTables = dbConn.QueryScalars($"SELECT name FROM {newDBName}.sqlite_schema WHERE type='table';").ToHashSet(); + for (int i = 0; i < unMergedTables.Count; i++) + { + var table = unMergedTables[i]; + if (newDBTables.Contains(table) || table == "configs") + { + continue; + } + unMergedTables.RemoveAt(i); + i--; + + // Then just tack them on + // Get command to create the table + logger.Info($"Adding table: {table}..."); + var createQuery = dbConn.ExecuteScalar($"SELECT sql FROM {oldDBName}.sqlite_schema WHERE type='table' AND name=?;", table); + + // Insert name of new database into create table query + createQuery = createQuery[..13] + newDBName + "." + createQuery[13..]; + logger.Debug($"Using command: {createQuery}"); + dbConn.Execute(createQuery); + + // Then add the rows + logger.Debug("Addings rows to table"); + dbConn.Execute($"INSERT INTO {newDBName}.{table} SELECT * FROM {oldDBName}.{table};"); + } + + logger.Info("Merging memos into new database..."); + MergeMemos(); + + logger.Info("Merging favorites into new database..."); + MergeFavorites(); + + logger.Info("Merging avatar and world cache into new database..."); + MergeCaches(); + + logger.Info("Merging gamelog into new database..."); + MergeGamelog(); + + logger.Info("Merging user feed tables into new database..."); + MergeUsers(); + + if (config.ImportConfig) + { + logger.Info("Importing config from old database..."); + ImportConfig(); + } + else + { + unMergedTables.Remove("configs"); + } + ResetDatabaseVersion(); + + foreach (var table in unMergedTables) + { + logger.Warn("Found unmerged table: " + table); + } + } + + private void MergeMemos() + { + MergeTable( + table => table.EndsWith("memos"), + [0], + (old, existing) => + { + if (existing == null) + { + logger.Trace("Inserting new memo"); + return old; + } + + logger.Trace("Replacing memo"); + + // Pick newer edited_at time + var oldDateTime = DateTime.Parse((string)old[1]); + var newDateTime = DateTime.Parse((string)existing[1]); + old[1] = oldDateTime > newDateTime ? oldDateTime : newDateTime; + old[1] = ((DateTime)old[1]).ToString(JSDateTimeFormat); + + // Don't concatenate memos if they're the exact same or + // the new memo ends with the old one (suggesting import + // has already been run) + old[2] = existing[2] == old[2] || ((string)existing[2]).EndsWith((string)old[2]) + ? existing[2] : old[2] + "\n" + existing[2]; + + return old; + } + ); + } + + private void MergeFavorites() + { + MergeTable( + table => table.StartsWith("favorite_"), + [2, 3], + (old, existing) => + { + if (existing == null) + { + logger.Trace("Inserting new favorite"); + // Let sqlite generate new pk + old[0] = null; + return old; + } + + logger.Trace("Replacing favorite"); + + // Favorites are the same, so just pick the older create + // time and add it + var oldDateTime = DateTime.Parse((string)old[1]); + var newDateTime = DateTime.Parse((string)existing[1]); + var updatedDateTime = oldDateTime < newDateTime ? oldDateTime : newDateTime; + existing[1] = updatedDateTime; + + return existing; + } + ); + } + + private void MergeCaches() + { + MergeTable( + table => table.StartsWith("cache_"), + [0], + (old, existing) => + { + if (existing == null) + { + logger.Trace("Inserting new cache entry"); + return old; + } + + logger.Trace("Replacing cache entry"); + + // old and existing have the same pk, so pick the newer + // cache entry + var oldDateTime = DateTime.Parse((string)old[1]); + var newDateTime = DateTime.Parse((string)existing[1]); + + return oldDateTime > newDateTime ? old : existing; + } + ); + } + + private void MergeGamelog() + { + // While this could be handled throw a single query, I would like to + // log anything determined to be a duplicate in case this doens't + // work + MergeTable( + table => table.StartsWith("gamelog_") && table != "gamelog_join_leave", + // These tables can be merged just fine by checking the created + // date and first col of information because we just need to + // know if 2 rows are the same + [1, 2], + // Literally just place back in what's already there + // created_at times should be pretty consistent, so we can trust + // that no duplicates will be created + (old, existing) => + { + if (existing != null) + { + logger.Trace("Determined these rows to be the same: "); + logger.Trace(string.Join(", ", old)); + logger.Trace(string.Join(", ", existing)); + } + old[0] = null; + // Return existing over old so we know that pk is unique + return existing ?? old; + }, + table => SortTable(dbConn, newDBName, table, 1) + ); + + MergeTable( + table => table == "gamelog_join_leave", + [1, 3], + (old, existing) => + { + if (existing != null) + { + logger.Trace("Determined these rows to be the same: "); + logger.Trace(string.Join(", ", old)); + logger.Trace(string.Join(", ", existing)); + } + old[0] = null; + return existing ?? old; + }, + table => SortTable(dbConn, newDBName, table, 1) + ); + } + + private void MergeUsers() + { + MergeTable( + table => userIDRegex.IsMatch(table) + && !table.EndsWith("_avatar_history") + && (table.EndsWith("_notifications") + || table.EndsWith("_moderation")), + [0], + (old, existing) => + { + if (existing == null) + { + logger.Trace("Inserting new feed entry"); + return old; + } + + logger.Trace("Replacing feed entry"); + + // old and existing have the same pk, so pick the newer + // cache entry + var oldDateTime = DateTime.Parse((string)old[1]); + var newDateTime = DateTime.Parse((string)existing[1]); + + return oldDateTime > newDateTime ? old : existing; + } + ); + + MergeTable( + table => userIDRegex.IsMatch(table) && table.EndsWith("_avatar_history"), + [0], + (old, existing) => + { + if (existing == null) + { + logger.Trace("Inserting new avatar feed entry"); + return old; + } + + logger.Trace("Replacing avatar feed entry"); + + // old and existing have the same pk, so pick the newer + // cache entry + var oldDateTime = DateTime.Parse((string)old[1]); + var newDateTime = DateTime.Parse((string)existing[1]); + + // Make sure to combine time values if they exist + if (existing.Length < 2) + { + return oldDateTime > newDateTime ? old : existing; + } + + var oldAvatarTime = old.Length >= 3 ? (int)old[2] : 0; + var newAvatarTime = (int)existing[2]; + + if (oldDateTime <= newDateTime) + { + old.CopyTo(existing, 0); + } + existing[2] = oldAvatarTime + newAvatarTime; + + logger.Trace( + "Combined avatar time: {} + {} = {}", + oldAvatarTime, newAvatarTime, oldAvatarTime + newAvatarTime + ); + + return existing; + } + ); + + var overlappingTables = new List(); + DateTime? oldestInNewTables = null; + DateTime? newestInOldTables = null; + for (int i = 0; i < unMergedTables.Count; i++) + { + // All other feed tables shouldve been merged, so just by + // matching user we should get all the rest of the user tables + string table = unMergedTables[i]; + if (!userIDRegex.IsMatch(table)) + { + continue; + } + unMergedTables.RemoveAt(i); + i--; + + // Skip friend log current for obvious reasons + if (table.EndsWith("_friend_log_current")) + { + continue; + } + + logger.Debug($"Merging table `{table}` into new database"); + + List colNames = GetTableColumnNames(dbConn, oldDBName, table); + + // Find min value of new db table and max value of old db table + var oldestInNew = dbConn.ExecuteScalar($"SELECT MIN({colNames[1]}) FROM {newDBName}.{table};"); + DateTime? oldestInNewDT = oldestInNew != null ? DateTime.Parse(oldestInNew) : null; + var newestInOld = dbConn.ExecuteScalar($"SELECT MAX({colNames[1]}) FROM {oldDBName}.{table};"); + DateTime? newestInOldDT = newestInOld != null ? DateTime.Parse(newestInOld) : null; + + // If either tables are empty or the oldest value in the new + // table is still newer than the newest value in the old + // (the tables don't overlap in time at all) + if (newestInOldDT == null || oldestInNewDT == null || oldestInNewDT > newestInOldDT) + { + logger.Debug($"User tables {table} has no overlap"); + // Then we can just combine them since there is no data + // overlap + var columnsClause = string.Join(", ", colNames.Skip(1)); + dbConn.Execute( + $"INSERT INTO {newDBName}.{table}({columnsClause})" + + $"SELECT {columnsClause} FROM {oldDBName}.{table};" + ); + SortTable(dbConn, newDBName, table, 1); + } + else + { + // I don't think people will actually care to choose a date + // for every single overlapping database + // Although it could easily be argued that people would want + // per user overlap control + logger.Debug($"User tables {table} has overlap"); + if (oldestInNewTables == null || oldestInNewDT < oldestInNewTables) + { + oldestInNewTables = oldestInNewDT; + } + if (newestInOldTables == null || newestInOldDT > newestInOldTables) + { + newestInOldTables = newestInOldDT; + } + overlappingTables.Add(table); + } + } + + if (overlappingTables.Count > 0) + { + // The datetimes should not be null here since there are + // overlapping tables + MergeUsersOverlap(overlappingTables, oldestInNewTables.Value, newestInOldTables.Value); + } + } + + private void MergeUsersOverlap(List tables, DateTime oldestInNew, DateTime newestInOld) + { + PrintOverlapWarning(tables, oldestInNew, newestInOld); + + string userInput = null; + bool datetimeParsed; + DateTime cutoffTime; + do + { + if (userInput != null) + { + logger.Error("Unrecognized input: " + userInput); + } + + userInput = Console.ReadLine(); + datetimeParsed = DateTime.TryParseExact( + userInput, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.None, out cutoffTime + ); + } + while (userInput != "keep old" && userInput != "keep new" && !datetimeParsed); + + // If user wants to keep new one then do nothing + if (userInput == "keep new") + { + logger.Info("Keeping new"); + return; + } + if (userInput == "keep old") + { + logger.Info("Keeping old"); + + // For old we just delete all rows from new and reinsert rows + // from old + foreach (var table in tables) + { + logger.Debug($"Deleting all rows in new database's {table}"); + dbConn.Execute($"DELETE FROM {newDBName}.{table}"); + + logger.Debug($"Adding rows from old database's {table}"); + dbConn.Execute($"INSERT INTO {newDBName}.{table} SELECT * FROM {oldDBName}.{table};"); + + SortTable(dbConn, newDBName, table, 1); + } + return; + } + + // Else we do the cutoff + var cutoffStr = cutoffTime.ToString(JSDateTimeFormat); + logger.Info("Merging at date: " + cutoffTime.ToString("yyyy-MM-dd HH:mm:ss")); + foreach (var table in tables) + { + // Get column names from old db in case newdb has columns + // old db doesn't + var colNames = GetTableColumnNames(dbConn, oldDBName, table); + + // Cutoff data in new db thats older than cutoff + logger.Debug($"Deleting rows in new database's {table} older than cutoff"); + dbConn.Execute($"DELETE FROM {newDBName}.{table} WHERE {colNames[1]} tables, DateTime oldestInNew, DateTime newestInOld) + { + var overlap = newestInOld - oldestInNew; + var overlapString = $"{overlap.Days} days, {overlap.Hours} hours"; + + logger.Warn(new string('=', 100)); + logger.Warn("WARNING:".PadLeft(46)); + logger.Warn("The merger has is unable to automatically merge the following USER FEED tables:"); + foreach (var table in tables) + { + logger.Warn(table); + } + logger.Warn(""); + logger.Warn("This is because these USER FEED tables contain overlap that can't be resolved:"); + logger.Warn("old database".PadRight(64, '-') + "|"); + logger.Warn(("|" + " new database".PadLeft(64, '-')).PadLeft(100)); + logger.Warn($"overlap ({overlapString})".PadRight(35) + "|" + new string('-', 28) + "|"); + logger.Warn("cutoff (^)".PadLeft(49)); + logger.Warn(""); + logger.Warn("Please choose a cutoff date and time. Data in the old USER FEED tables after this date"); + logger.Warn("will be discarded, and data in the new USER FEED tables before this date will be discarded as well."); + logger.Warn("The remaining data will then be spliced together."); + logger.Warn("Your input should be in this format: `YYYY-MM-DD HH:MM:SS`"); + logger.Warn(""); + logger.Warn("Alternatively, you can enter `keep new` to discard the old data or `keep old` to discard new data."); + logger.Warn(""); + logger.Warn("Again, this only affects USER FEED tables, all other tables are merged properly."); + logger.Warn("To read more about this process please check the VRCX wiki: "); + logger.Warn(new string('=', 100)); + } + + private void ImportConfig() + { + unMergedTables.Remove("configs"); + + // Skip saved credentials to avoid accidentally exposing sensitive + // information somehow + dbConn.Execute( + $"INSERT OR REPLACE INTO {newDBName}.configs " + + $"SELECT * FROM {oldDBName}.configs " + + $"WHERE key!=?;", "config:savedcredentials" + ); + } + + private void ResetDatabaseVersion() + { + // Tell VRCX to add in any missing fields that the merger may have + // missed, just as a precaution + dbConn.Execute( + $"INSERT OR REPLACE INTO {newDBName}.configs VALUES (?, 0)", + "config:vrcx_databaseversion" + ); + } + + /// + /// A method that automates various processes of merging. + /// + /// It first finds a table that matches the `tableMatcher` predicate, + /// then removes it from `unMergedTables`. + /// Then it loops over every row in the old database table, checking if + /// the row exists in the new table. It does this by checking if the + /// column indices passed into `colIndicesToMatch` are the same. + /// Then for each row, it calls `rowTransformer`, passing in the old + /// rows and existing new rows. `rowTransformer` should return the row + /// to insert into the new database or null. + /// + /// A predicate to check if a table is one to edit + /// The column indices to match to see if old and new rows are the same + /// An func called on every row + /// An action called once all rows have been iterated + private void MergeTable( + Predicate tableMatcher, + int[] colIndicesToMatch, + Func rowTransformer, + Action finalizer = null + ) { + for (int i = 0; i < unMergedTables.Count; i++) + { + // Find table that we want to merge + string table = unMergedTables[i]; + if (!tableMatcher(table)) + { + continue; + } + unMergedTables.RemoveAt(i); + i--; + + logger.Debug($"Merging table `{table}` into new database"); + + // Prepare queries + var colNames = GetTableColumnNames(dbConn, newDBName, table); + + var valuesClause = string.Join(',', new string('?', colNames.Count).ToCharArray()); + var insertQuery = $"INSERT INTO {newDBName}.{table} VALUES ({valuesClause});"; + + var whereClause = string.Join(" AND ", colIndicesToMatch.Select(i => colNames[i] + "=?")); + var existsQuery = $"DELETE FROM {newDBName}.{table} WHERE {whereClause} RETURNING *;"; + + // Loop over every row on table in old database + var rowsCommand = dbConn.CreateCommand($"SELECT * FROM {oldDBName}.{table};"); + foreach (object[] oldRow in rowsCommand.ExecuteQueryScalars(dbConn)) + { + // Find existing row (if it exists at all) and remove it + object[] colsToMatch = colIndicesToMatch.Select(i => oldRow[i]).ToArray(); + var existingRow = dbConn.QueryScalars(existsQuery, colsToMatch).FirstOrDefault(); + + // Insert new row in place of the existing row + var newRow = rowTransformer(oldRow, existingRow); + if (newRow.Length < colNames.Count) + { + // Row count may not match between old and new db, so + // just add null data + var temp = new object[colNames.Count]; + newRow.CopyTo(temp, 0); + newRow = temp; + } + dbConn.Execute(insertQuery, newRow); + } + + finalizer?.Invoke(table); + } + } + + private static List GetTableColumnNames(SQLiteConnection conn, string db, string table) + => conn.QueryScalars($"SELECT name FROM pragma_table_info(?, ?);", table, db); + + private static void SortTable(SQLiteConnection conn, string db, string table, int sortCol, bool isDesc = false) + { + logger.Debug($"Sorting table {db}.{table}"); + + // Just to ensure name is unique + var time = DateTime.Now.Ticks; + var newTableName = table + time.ToString(); + logger.Debug($"Creating new table: " + newTableName); + + // Split create query into words + var createQuery = conn.ExecuteScalar($"SELECT sql FROM {db}.sqlite_schema WHERE type='table' AND name=?;", table); + var words = createQuery.Split(' '); + // Third word is table name + words[2] = db + "." + newTableName; + createQuery = string.Join(' ', words); + + logger.Debug("Creating table with command: " + createQuery); + conn.Execute(createQuery); + + logger.Debug("Adding rows..."); + var colNames = GetTableColumnNames(conn, db, table); + // Skip pks so they get reassigned + var selectClause = string.Join(',', colNames.Skip(1)); + conn.Execute( + $"INSERT INTO {db}.{newTableName} ({selectClause}) " + + $"SELECT {selectClause} FROM {db}.{table} " + + $"ORDER BY {colNames[sortCol]} {(isDesc ? "DESC" : "ASC")};" + ); + + logger.Debug("Dropping old and renaming"); + conn.Execute($"DROP TABLE {db}.{table}"); + conn.Execute($"ALTER TABLE {db}.{newTableName} RENAME TO {table}"); + } + + [GeneratedRegex("^([A-Za-z0-9]{10}|usr[0-9A-Fa-f]{32})")] + private static partial Regex UserIDRegex(); + } +} diff --git a/DBMerger/Program.cs b/DBMerger/Program.cs new file mode 100644 index 00000000..65709259 --- /dev/null +++ b/DBMerger/Program.cs @@ -0,0 +1,209 @@ +using NLog; +using NLog.Targets; +using SQLite; +using System; + +// Use different command line parser for more standardized output +// (like help text) +using System.CommandLine; +using System.CommandLine.Parsing; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; + +namespace DBMerger +{ + public class Program + { + private static readonly Logger logger = LogManager.GetLogger("DBMerger"); + + // TODO: Consider config class? + public static SQLiteConnection DBConn { get; private set; } + public static SQLiteConnection OldDBConn { get; private set; } + public static Config Config { get; private set; } + + public static void Main(string[] args) + { + ProcessArgs(args); + + ConfigureLogger(); + + if (Config.Debug) + { + // Needed? mostly just covering my ass + logger.Warn(new string('=', 100)); + logger.Warn("WARNING:".PadLeft(46)); + logger.Warn("Debug mode will output some sensitive information (friends list, friend history, etc.)"); + logger.Warn("Only use this mode for debug purposes. Enter `y` to confirm or anything else to exit."); + logger.Warn(new string('=', 100)); + if (Console.ReadLine() != "y") + { + return; + } + } + + var asm = Assembly.GetExecutingAssembly(); + var versionInfo = FileVersionInfo.GetVersionInfo(asm.Location); + logger.Info($"{versionInfo.ProductName}-{versionInfo.ProductVersion}"); + logger.Info($"by {versionInfo.LegalCopyright}\n"); + + if (Path.GetFullPath(Config.NewDBPath) == Path.GetFullPath(Config.OldDBPath)) + { + logger.Fatal("Database pathes cannot be the same!"); + return; + } + + try + { + logger.Debug("Creating connection to old DB"); + try + { + DBConn = new SQLiteConnection(Config.OldDBPath) { Tracer = logger.Trace, Trace = true }; + } + catch (SQLiteException) + { + logger.Fatal("Could not connect to old DB. Perhaps passed in db is corrupt or not a valid sqlite db?"); + return; + } + + logger.Debug("Creating connection to new DB"); + try + { + DBConn.Execute("ATTACH DATABASE ? AS new_db", Config.NewDBPath); + } + catch (SQLiteException) + { + logger.Fatal("Could not connect to new DB. Perhaps passed in db is corrupt or not a valid sqlite db?"); + return; + } + logger.Info("Database connections created successfully!"); + + CreateBackup(); + + try + { + new Merger(DBConn, "main", "new_db", Config).Merge(); + } + catch (Exception ex) + { + logger.Fatal(ex, "Merge process failed with error:\n"); + } + } + finally + { + logger.Debug("Closing database connection..."); + DBConn.Close(); + } + + } + + private static void ProcessArgs(string[] args) + { + static string validateDBPath(ArgumentResult arg) + { + string path = arg.Tokens[0].Value; + Option option = arg.Argument.Parents.Single() as Option; + + string extension = Path.GetExtension(path); + if (extension == "") + { + path += Path.ChangeExtension(path, "sqlite3"); + } + else if (extension != ".sqlite3") + { + arg.ErrorMessage = $"File given to option `{option.Aliases.First()}` is not a sqlite database!"; + return null; + } + + if (!File.Exists(path)) + { + arg.ErrorMessage = $"File given to option `{option.Aliases.First()}` does not exist!"; + return null; + } + + return path; + } + + var rootCommand = new RootCommand("Merge an old and new VRCX sqlite database into one."); + + var newDBOption = new Option( + ["-n", "--new-db-path"], + description: "The path of the new DB to merge the old onto.", + parseArgument: validateDBPath + ) { IsRequired = true }; + rootCommand.AddOption(newDBOption); + + var oldDBOption = new Option( + ["-o", "--old-db-path"], + description: "The path of the old DB to merge into the new.", + parseArgument: validateDBPath + ) { IsRequired = true }; + rootCommand.AddOption(oldDBOption); + + // Add `debug` option to be consistent with args from the main exe + var debugOption = new Option(["-v", "--verbose", "-d", "--debug"], () => false, "Add debug information to the output."); + rootCommand.AddOption(debugOption); + + var importConfigOption = new Option(["--import-config"], () => false, "Imports the config values from the old database. This will override the config in the new database."); + rootCommand.AddOption(importConfigOption); + + rootCommand.SetHandler((newDBPath, oldDBPath, debug, importConfig) => + { + Config = new Config(newDBPath, oldDBPath, debug, importConfig); + }, newDBOption, oldDBOption, debugOption, importConfigOption); + + // If the args weren't parsable or verifiable, exit + if (rootCommand.Invoke(args) != 0) + { + Environment.Exit(0); + } + } + + private static void ConfigureLogger() + { + LogManager.Setup().LoadConfiguration(builder => + { + var fileTarget = new FileTarget("fileTarget") + { + FileName = "DBMerger.log", + Layout = "${longdate} [${level:uppercase=true:padding=-5}] ${logger:padding=-20} - ${message} ${exception:format=tostring}", + ArchiveFileName = Path.Combine("DBMerger_Logs", "DBMerger.{#}.log"), + ArchiveNumbering = ArchiveNumberingMode.DateAndSequence, + ArchiveOldFileOnStartup = true, + ConcurrentWrites = true, + KeepFileOpen = true, + AutoFlush = true, + Encoding = System.Text.Encoding.UTF8 + }; + + var consoleTarget = new ColoredConsoleTarget() + { + Layout = "[${level:uppercase=true:padding=-5}] ${message} ${exception:format=tostring}", + AutoFlush = true, + Encoding = System.Text.Encoding.UTF8 + }; + + builder.ForLogger().FilterMinLevel(Config.Debug ? LogLevel.Trace : LogLevel.Debug).WriteTo(fileTarget); + builder.ForLogger().FilterMinLevel(Config.Debug ? LogLevel.Trace : LogLevel.Info).WriteTo(consoleTarget); + }); + } + + private static void CreateBackup() + { + // Get unique name for backup. Format matches the log file name format + string date = DateTime.Now.ToString("yyyyMMdd"); + int counter = 0; + string backupPath; + do + { + backupPath = Path.Combine(Path.GetDirectoryName(Config.NewDBPath), $"VRCX.back.{date}.{counter}.sqlite3"); + counter++; + } + while (File.Exists(backupPath)); + + File.Copy(Config.NewDBPath, backupPath); + logger.Info($"Created backup of new DB at {backupPath}"); + } + } +} diff --git a/DBMerger/SqliteExtensions.cs b/DBMerger/SqliteExtensions.cs new file mode 100644 index 00000000..066f6b5d --- /dev/null +++ b/DBMerger/SqliteExtensions.cs @@ -0,0 +1,77 @@ +using SQLite; +using SQLitePCL; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace DBMerger +{ + // This class is made of mostly hardcoded copies from the sqlite lib. + // Normally this would be very bad, but since the library is long since + // unmaintained it shouldn't matter + internal static class SqliteExtensions + { + // The prepare method is private, so fetch it here + private static readonly MethodInfo _prepareMethod = typeof(SQLiteCommand).GetMethod("Prepare", BindingFlags.NonPublic | BindingFlags.Instance); + + /// + /// Creates a SQLiteCommand given the command text (SQL) with arguments. Place a '?' + /// in the command text for each of the arguments and then executes that command. + /// It returns each row as an array of object primitives. + /// + /// + /// The fully escaped SQL. + /// + /// + /// Arguments to substitute for the occurences of '?' in the query. + /// + /// + /// An enumerable with one object array for each row. + /// + public static List QueryScalars(this SQLiteConnection conn, string query, params object[] args) + { + var cmd = conn.CreateCommand(query, args); + return cmd.ExecuteQueryScalars(conn).ToList(); + } + + public static IEnumerable ExecuteQueryScalars(this SQLiteCommand cmd, SQLiteConnection conn) + { + if (conn.Trace) + { + conn.Tracer?.Invoke("Executing Query: " + cmd); + } + var stmt = _prepareMethod.Invoke(cmd, []) as sqlite3_stmt; + try + { + int columnCount = SQLite3.ColumnCount(stmt); + if (SQLite3.ColumnCount(stmt) < 1) + { + throw new InvalidOperationException("QueryScalars should return at least one column"); + } + while (SQLite3.Step(stmt) == SQLite3.Result.Row) + { + var row = new object[columnCount]; + for (int i = 0; i < columnCount; i++) + { + var colType = SQLite3.ColumnType(stmt, i); + row[i] = colType switch + { + SQLite3.ColType.Integer => SQLite3.ColumnInt(stmt, i), + SQLite3.ColType.Float => (float)SQLite3.ColumnDouble(stmt, i), + SQLite3.ColType.Text => SQLite3.ColumnString(stmt, i), + SQLite3.ColType.Blob => SQLite3.ColumnByteArray(stmt, i), + SQLite3.ColType.Null or _ => null + }; + } + yield return row; + } + } + finally + { + SQLite3.Finalize(stmt); + } + } + + } +} diff --git a/Dotnet/VRCX-Cef.csproj b/Dotnet/VRCX-Cef.csproj index a0966de1..7771fca1 100644 --- a/Dotnet/VRCX-Cef.csproj +++ b/Dotnet/VRCX-Cef.csproj @@ -5,7 +5,7 @@ WinExe net9-windows10.0.19041.0 - x64 + x64 true 0 1.0.0.%2a @@ -46,7 +46,7 @@ true - + win-x64 false @@ -59,6 +59,11 @@ false false + + + + + libs\Blake2Sharp.dll diff --git a/Dotnet/VRCX-Electron.csproj b/Dotnet/VRCX-Electron.csproj index c6224a61..0be9a6ce 100644 --- a/Dotnet/VRCX-Electron.csproj +++ b/Dotnet/VRCX-Electron.csproj @@ -7,7 +7,7 @@ net9.0 Library - x64 + x64 0 1.0.0.%2a VRCX @@ -46,8 +46,8 @@ ..\VRCX.ico - - + + win-x64 false @@ -125,5 +125,7 @@ + + \ No newline at end of file diff --git a/VRCX.sln b/VRCX.sln index 3219f7f5..8debcde1 100644 --- a/VRCX.sln +++ b/VRCX.sln @@ -12,6 +12,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VRCX-Electron", "Dotnet\VRCX-Electron.csproj", "{B0275E4A-FE0F-410A-96F1-1D73DF2535FA}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DBMerger", "DBMerger\DBMerger.csproj", "{9BE1DD2F-CABC-4CF9-A53E-C62923760887}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -26,6 +28,10 @@ Global {B0275E4A-FE0F-410A-96F1-1D73DF2535FA}.Debug|x64.Build.0 = Debug|x64 {B0275E4A-FE0F-410A-96F1-1D73DF2535FA}.Release|x64.ActiveCfg = Release|x64 {B0275E4A-FE0F-410A-96F1-1D73DF2535FA}.Release|x64.Build.0 = Release|x64 + {9BE1DD2F-CABC-4CF9-A53E-C62923760887}.Debug|x64.ActiveCfg = Debug|Any CPU + {9BE1DD2F-CABC-4CF9-A53E-C62923760887}.Debug|x64.Build.0 = Debug|Any CPU + {9BE1DD2F-CABC-4CF9-A53E-C62923760887}.Release|x64.ActiveCfg = Release|Any CPU + {9BE1DD2F-CABC-4CF9-A53E-C62923760887}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE