Fix issues with Journal handling

Implement basic database
Handle startup events
only send events after the most recent LoadGame

Pulsar/Features/Backpack/BackpackService.cs

@@ -0,0 +1,46 @@
+namespace Pulsar.Features.Backpack;
+
+using Observatory.Framework.Files;
+
+public interface IBackpackService : IJournalHandler<BackpackFile>;
+
+public class BackpackService(IOptions<PulsarConfiguration> options, IEventHubContext hub, ILogger<BackpackService> logger) : IBackpackService
+{
+    public async Task<BackpackFile> Get()
+    {
+        var filePath = Path.Combine(options.Value.JournalDirectory, FileName);
+
+        if (!FileHelper.ValidateFile(filePath))
+        {
+            return new BackpackFile();
+        }
+
+        await using var file = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+        var backpack = await JsonSerializer.DeserializeAsync<BackpackFile>(file);
+        if (backpack != null) return backpack;
+
+        logger.LogWarning("Failed to deserialize backpack file {File}", filePath);
+        return new BackpackFile();
+    }
+
+    public async Task HandleFile(string path, CancellationToken token = new ())
+    {
+        if (!FileHelper.ValidateFile(path))
+        {
+            return;
+        }
+
+        var file = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+        var backpack = await JsonSerializer.DeserializeAsync<BackpackFile>(file, cancellationToken: token);
+
+        if (backpack == null)
+        {
+            logger.LogWarning("Failed to deserialize backpack {FilePath}", file);
+            return;
+        }
+
+        await hub.Clients.All.BackpackUpdated(backpack);
+    }
+
+    public string FileName => FileHandlerService.BackpackFileName;
+}

Pulsar/Features/FileWatcherService.cs

@@ -3,8 +3,8 @@ namespace Pulsar.Features;
 using System.Collections.Concurrent;
 using Microsoft.Extensions.FileProviders;
 
-public class FileWatcherService(IOptions<PulsarConfiguration> options, IFileHandlerService fileHandlerService)
-    : IHostedService
+public class FileWatcherService(IOptions<PulsarConfiguration> options, IFileHandlerService fileHandlerService, ILogger<FileWatcherService> logger)
+    : IHostedService, IDisposable
 {
     private PhysicalFileProvider watcher = null!;
 
@@ -20,11 +20,8 @@ public class FileWatcherService(IOptions<PulsarConfiguration> options, IFileHand
 
         // read the journal directory to get the initial files
 #if DEBUG
-        Task.Run(() =>
-        {
-            Thread.Sleep(TimeSpan.FromSeconds(2));
-            HandleFileChanged(cancellationToken);
-        }, cancellationToken);
+        Thread.Sleep(TimeSpan.FromSeconds(2));
+        HandleFileChanged(cancellationToken);
 #else
         HandleFileChanged(cancellationToken);
 #endif
@@ -37,35 +34,46 @@ public class FileWatcherService(IOptions<PulsarConfiguration> options, IFileHand
 
     private void HandleFileChanged(CancellationToken token = new())
     {
+        Watch(token);
         var tasks = new List<Task>();
-        foreach (var file in watcher.GetDirectoryContents(""))
+        try
         {
-            if (file.IsDirectory || !file.Name.EndsWith(".json") &&
-                !(file.Name.StartsWith(FileHandlerService.JournalLogFileNameStart) &&
-                  file.Name.EndsWith(FileHandlerService.JournalLogFileNameEnd)))
+            foreach (var file in watcher.GetDirectoryContents(""))
             {
-                return;
-            }
-
-
-            FileDates.AddOrUpdate(file.PhysicalPath, _ =>
-            {
-                tasks.Add(Task.Run(() => fileHandlerService.HandleFile(file.PhysicalPath, token), token));
-                return file.LastModified;
-            }, (_, existing) =>
-            {
-                if (existing != file.LastModified)
+                logger.LogDebug("Checking File: {File}", file.PhysicalPath);
+                if (file.IsDirectory || (!file.Name.EndsWith(".json") &&
+                                         !(file.Name.StartsWith(FileHandlerService.JournalLogFileNameStart) &&
+                                           file.Name.EndsWith(FileHandlerService.JournalLogFileNameEnd))))
                 {
-                    tasks.Add(Task.Run(() => fileHandlerService.HandleFile(file.PhysicalPath, token), token));
+                    continue;
                 }
 
-                return file.LastModified;
-            });
-        }
+                logger.LogDebug("Has File Updated?: {File}, {LastModified}", file.PhysicalPath, file.LastModified);
 
-        Watch(token);
-        
-        Task.WaitAll(tasks.ToArray(), token);
+                FileDates.AddOrUpdate(file.PhysicalPath, _ =>
+                {
+                    logger.LogDebug("New File: {File}", file.PhysicalPath);
+                    tasks.Add(Task.Run(() => fileHandlerService.HandleFile(file.PhysicalPath, token), token));
+                    return file.LastModified;
+                }, (_, existing) =>
+                {
+                    logger.LogDebug("Existing File: {File}", file.PhysicalPath);
+                    if (existing != file.LastModified)
+                    {
+                        logger.LogDebug("File Updated: {File}", file.PhysicalPath);
+                        tasks.Add(Task.Run(() => fileHandlerService.HandleFile(file.PhysicalPath, token), token));
+                    }
+
+                    return file.LastModified;
+                });
+            }
+
+            Task.WaitAll(tasks.ToArray(), token);
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Error handling file change");
+        }
     }
 
     private void Watch(CancellationToken token)
@@ -75,7 +83,14 @@ public class FileWatcherService(IOptions<PulsarConfiguration> options, IFileHand
             HandleFileChanged(token);
         }
 
-        watcher.Watch("*.*").RegisterChangeCallback(Handle, null);
+        try
+        {
+            watcher.Watch("*.*").RegisterChangeCallback(Handle, null);
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Error watching directory {Directory}", watcher.Root);
+        }
     }
 
     public Task StopAsync(CancellationToken cancellationToken)
@@ -83,4 +98,9 @@ public class FileWatcherService(IOptions<PulsarConfiguration> options, IFileHand
         watcher.Dispose();
         return Task.CompletedTask;
     }
+
+    public void Dispose()
+    {
+        watcher.Dispose();
+    }
 }

Pulsar/Features/Journal/JournalProcessor.cs

@@ -1,3 +1,5 @@
+using Observatory.Framework.Files.Journal.Startup;
+
 namespace Pulsar.Features.Journal;
 
 using Observatory.Framework;
@@ -6,7 +8,8 @@ using Observatory.Framework.Files.Journal;
 public class JournalProcessor(
     ILogger<JournalProcessor> logger,
     IJournalService journalService,
-    IEventHubContext hub) : IJournalProcessor, IHostedService, IDisposable
+    PulsarContext context,
+    IEventHubContext hub) : IHostedService, IDisposable
 {
     private CancellationTokenSource tokenSource = new();
     
@@ -18,7 +21,7 @@ public class JournalProcessor(
         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals,
     };
 
-    public async Task HandleFileInner(string filePath, CancellationToken token = new())
+    public async Task<List<JournalBase>> HandleFileInner(string filePath, CancellationToken token = new())
     {
         logger.LogInformation("Processing journal file: '{File}'", filePath);
         var file = await File.ReadAllBytesAsync(filePath, token);
@@ -49,8 +52,20 @@ public class JournalProcessor(
                     continue;
                 }
 
+                if (journal is LoadGame loadGame)
+                {
+                    // if not existing, add
+                    if (context.LoadGames.Any(l => l.Timestamp == loadGame.Timestamp))
+                    {
+                        //return ValueTask.CompletedTask;
+                        continue;
+                    }
+                    await context.LoadGames.AddAsync(loadGame, token);
+                    await context.SaveChangesAsync(token);
+                }
+
                 newJournals.Add(journal);
-            }
+            }   
             catch (JsonException ex)
             {
                 logger.LogError(ex, "Error deserializing journal file: '{File}', line: {Line}", filePath, line);
@@ -59,11 +74,7 @@ public class JournalProcessor(
             //return ValueTask.CompletedTask;
         }
 
-
-        if (newJournals.Any())
-        {
-            await hub.Clients.All.JournalUpdated(newJournals);
-        }
+        return newJournals;
     }
 
     public Task StartAsync(CancellationToken cancellationToken)
@@ -78,15 +89,36 @@ public class JournalProcessor(
     {
         Task.Run(async () =>
         {
-            while (!tokenSource.Token.IsCancellationRequested)
+            var token = tokenSource.Token;
+            var handled = new List<JournalBase>();
+            while (!token.IsCancellationRequested)
             {
-                if (journalService.TryDequeue(out var file))
+                try
                 {
-                    await HandleFileInner(file, tokenSource.Token);
+                    if (journalService.TryDequeue(out var file))
+                    {
+                        handled.AddRange(await HandleFileInner(file, tokenSource.Token));
+                    }
+                    else if (handled.Count > 0)
+                    {
+                        //get last loadgame
+                        var lastLoadGame = context.LoadGames.OrderByDescending(l => l.Timestamp).FirstOrDefault();
+                        // only emit journals since last loadgame
+                        if (lastLoadGame != null)
+                        {
+                            handled = handled.Where(j => j.Timestamp > lastLoadGame.Timestamp).ToList();
+                        }
+                        await hub.Clients.All.JournalUpdated(handled);
+                        handled.Clear();
+                    }
+                    else
+                    {
+                        await Task.Delay(1000, token);
+                    }
                 }
-                else
+                catch (Exception ex)
                 {
-                    await Task.Delay(1000);
+                    logger.LogError(ex, "Error processing journal queue");
                 }
             }
         }, tokenSource.Token);
@@ -103,7 +135,3 @@ public class JournalProcessor(
         tokenSource?.Dispose();
     }
 }
-
-public interface IJournalProcessor
-{
-}

Pulsar/Features/Market/MarketService.cs

@@ -0,0 +1,46 @@
+namespace Pulsar.Features.Market;
+
+using Observatory.Framework.Files;
+
+public interface IMarketService : IJournalHandler<MarketFile>;
+
+public class MarketService(IOptions<PulsarConfiguration> options, IEventHubContext hub, ILogger<MarketService> logger) : IMarketService
+{
+    public async Task<MarketFile> Get()
+    {
+        var filePath = Path.Combine(options.Value.JournalDirectory, FileName);
+
+        if (!FileHelper.ValidateFile(filePath))
+        {
+            return new MarketFile();
+        }
+
+        await using var file = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+        var market = await JsonSerializer.DeserializeAsync<MarketFile>(file);
+        if (market != null) return market;
+
+        logger.LogWarning("Failed to deserialize market file {File}", filePath);
+        return new MarketFile();
+    }
+
+    public async Task HandleFile(string path, CancellationToken token = new ())
+    {
+        if (!FileHelper.ValidateFile(path))
+        {
+            return;
+        }
+
+        var file = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+        var market = await JsonSerializer.DeserializeAsync<MarketFile>(file, cancellationToken: token);
+
+        if (market == null)
+        {
+            logger.LogWarning("Failed to deserialize market File {FilePath}", file);
+            return;
+        }
+
+        await hub.Clients.All.MarketUpdated(market);
+    }
+
+    public string FileName => FileHandlerService.MarketFileName;
+}

Pulsar/Features/NavRoute/NavRouteService.cs

@@ -0,0 +1,46 @@
+namespace Pulsar.Features.NavRoute;
+
+using Observatory.Framework.Files;
+
+public interface INavRouteService :  IJournalHandler<NavRouteFile>;
+
+public class NavRouteService(IOptions<PulsarConfiguration> options, ILogger<NavRouteService> logger, IEventHubContext hub) : INavRouteService
+{
+    public async Task<NavRouteFile> Get()
+    {
+        var navRouteFile = Path.Combine(options.Value.JournalDirectory, FileName);
+
+        if (!FileHelper.ValidateFile(navRouteFile))
+        {
+            return new NavRouteFile();
+        }
+
+        await using var file = File.Open(navRouteFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+        var shipLocker = await JsonSerializer.DeserializeAsync<NavRouteFile>(file);
+        if (shipLocker != null) return shipLocker;
+
+        logger.LogWarning("Failed to deserialize nav route file {ShipLockerFile}", navRouteFile);
+        return new NavRouteFile();
+    }
+
+    public async Task HandleFile(string path, CancellationToken token = new ())
+    {
+        if (!FileHelper.ValidateFile(path))
+        {
+            return;
+        }
+
+        var file = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+        var navRoute = await JsonSerializer.DeserializeAsync<NavRouteFile>(file, cancellationToken: token);
+
+        if (navRoute == null)
+        {
+            logger.LogWarning("Failed to deserialize nav route {FilePath}", file);
+            return;
+        }
+
+        await hub.Clients.All.NavRouteUpdated(navRoute);
+    }
+
+    public string FileName => FileHandlerService.NavRouteFileName;
+}

Pulsar/Features/Outfitting/OutfittingService.cs

@@ -0,0 +1,46 @@
+namespace Pulsar.Features.Outfitting;
+
+using Observatory.Framework.Files;
+
+public interface IOutfittingService : IJournalHandler<OutfittingFile>;
+
+public class OutfittingService(IOptions<PulsarConfiguration> options, IEventHubContext hub, ILogger<OutfittingService> logger) : IOutfittingService
+{
+    public async Task<OutfittingFile> Get()
+    {
+        var filePath = Path.Combine(options.Value.JournalDirectory, FileName);
+
+        if (!FileHelper.ValidateFile(filePath))
+        {
+            return new OutfittingFile();
+        }
+
+        await using var file = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+        var outfitting = await JsonSerializer.DeserializeAsync<OutfittingFile>(file);
+        if (outfitting != null) return outfitting;
+
+        logger.LogWarning("Failed to deserialize outfitting file {File}", filePath);
+        return new OutfittingFile();
+    }
+
+    public async Task HandleFile(string path, CancellationToken token = new ())
+    {
+        if (!FileHelper.ValidateFile(path))
+        {
+            return;
+        }
+
+        var file = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+        var outfitting = await JsonSerializer.DeserializeAsync<OutfittingFile>(file, cancellationToken: token);
+
+        if (outfitting == null)
+        {
+            logger.LogWarning("Failed to deserialize outfitting file {FilePath}", file);
+            return;
+        }
+
+        await hub.Clients.All.OutfittingUpdated(outfitting);
+    }
+
+    public string FileName => FileHandlerService.OutfittingFileName;
+}