Update Journal File Handling

Now Correctly deserializes Journal events

ObservatoryFramework/Files/Converters/FleetCarrierTravelConverter.cs

@@ -7,13 +7,13 @@ class FleetCarrierTravelConverter : JsonConverter<float>
 {
     public override float Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
     {
-            if (reader.TokenType == JsonTokenType.String)
+        if (reader.TokenType == JsonTokenType.String)
-                return float.Parse(reader.GetString().Split(' ')[0]);
+            return float.Parse(reader.GetString().Split(' ')[0]);
-            return reader.GetSingle();
+        return reader.GetSingle();
-        }
+    }
 
     public override void Write(Utf8JsonWriter writer, float value, JsonSerializerOptions options)
     {
-            writer.WriteStringValue(value.ToString());
+        writer.WriteStringValue(value.ToString());
-        }
+    }
 }

ObservatoryFramework/Files/Converters/IntBoolFlexConverter.cs

@@ -7,13 +7,13 @@ public class IntBoolFlexConverter : JsonConverter<bool>
 {
     public override bool Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
     {
-            if (reader.TokenType == JsonTokenType.Number)
+        if (reader.TokenType == JsonTokenType.Number)
-                return reader.GetInt16() == 1;
+            return reader.GetInt16() == 1;
-            return reader.GetBoolean();
+        return reader.GetBoolean();
-        }
+    }
 
     public override void Write(Utf8JsonWriter writer, bool value, JsonSerializerOptions options)
     {
-            writer.WriteBooleanValue(value);
+        writer.WriteBooleanValue(value);
-        }
+    }
 }

ObservatoryFramework/Files/Converters/LegacyFactionConverter.cs

@@ -21,6 +21,6 @@ public class LegacyFactionConverter<TFaction> : JsonConverter<TFaction> where TF
 
     public override void Write(Utf8JsonWriter writer, TFaction value, JsonSerializerOptions options)
     {
-            throw new NotImplementedException();
+        JsonSerializer.Serialize(writer, value, options);
-        }
+    }
 }

ObservatoryFramework/Files/Converters/MaterialCompositionConverter.cs

@@ -41,11 +41,12 @@ public class MaterialCompositionConverter : JsonConverter<ImmutableList<Material
             return materialComposition.ToImmutableList();
         }
 
-        return (ImmutableList<MaterialComposition>)JsonSerializer.Deserialize(ref reader, typeof(ImmutableList<MaterialComposition>));
+        return JsonSerializer.Deserialize<ImmutableList<MaterialComposition>>(ref reader, options)!;
     }
 
-    public override void Write(Utf8JsonWriter writer, ImmutableList<MaterialComposition> value, JsonSerializerOptions options)
+    public override void Write(Utf8JsonWriter writer, ImmutableList<MaterialComposition> value,
+        JsonSerializerOptions options)
     {
-        throw new NotImplementedException();
+        JsonSerializer.Serialize(writer, value, options);
     }
 }

ObservatoryFramework/Files/Converters/MaterialConverter.cs

@@ -11,42 +11,44 @@ namespace Observatory.Framework.Files.Converters;
 /// </summary>
 public class MaterialConverter : JsonConverter<ImmutableList<Material>>
 {
-    public override ImmutableList<Material> Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    public override ImmutableList<Material> Read(ref Utf8JsonReader reader, Type typeToConvert,
+        JsonSerializerOptions options)
     {
-            if (reader.TokenType == JsonTokenType.StartObject)
+        if (reader.TokenType == JsonTokenType.StartObject)
+        {
+            var materialComposition = new List<Material>();
+            while (reader.Read())
             {
-                var materialComposition = new List<Material>();
+                if (reader.TokenType != JsonTokenType.EndObject)
-                while (reader.Read())
                 {
-                    if (reader.TokenType != JsonTokenType.EndObject)
+                    if (reader.TokenType == JsonTokenType.PropertyName)
                     {
-                        if (reader.TokenType == JsonTokenType.PropertyName)
+                        var name = reader.GetString();
+                        reader.Read();
+                        var count = reader.GetInt32();
+                        var material = new Material
                         {
-                            var name = reader.GetString();
+                            Name = name,
-                            reader.Read();
+                            Name_Localised = name,
-                            var count = reader.GetInt32();
+                            Count = count
-                            var material = new Material
+                        };
-                            {
+                        materialComposition.Add(material);
-                                Name = name,
-                                Name_Localised = name,
-                                Count = count
-                            };
-                            materialComposition.Add(material);
-                        }
-                    }
-                    else
-                    {
-                        break;
                     }
                 }
-                return materialComposition.ToImmutableList();
+                else
+                {
+                    break;
+                }
             }
 
-            return (ImmutableList<Material>)JsonSerializer.Deserialize(ref reader, typeof(ImmutableList<Material>));
+            return materialComposition.ToImmutableList();
         }
 
+        return JsonSerializer.Deserialize<ImmutableList<Material>>(ref reader, options)!;
+    }
+
     public override void Write(Utf8JsonWriter writer, ImmutableList<Material> value, JsonSerializerOptions options)
     {
-            throw new NotImplementedException();
+        JsonSerializer.Serialize(writer, value, options);
-        }
+    }
 }

ObservatoryFramework/Files/Converters/MissionEffectConverter.cs

@@ -32,6 +32,6 @@ public class MissionEffectConverter : JsonConverter<MissionEffect>
 
     public override void Write(Utf8JsonWriter writer, MissionEffect value, JsonSerializerOptions options)
     {
-        throw new NotImplementedException();
+        JsonSerializer.Serialize(writer, value, options);
     }
 }

ObservatoryFramework/Files/Converters/MutableStringDoubleConverter.cs

@@ -14,6 +14,9 @@ class MutableStringDoubleConverter : JsonConverter<object>
 
     public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options)
     {
-        throw new NotImplementedException();
+        if (value.GetType() == typeof(string))
+            writer.WriteStringValue((string)value);
+        else
+            writer.WriteNumberValue((double)value);
     }
 }

ObservatoryFramework/Files/Converters/PipConverter.cs

@@ -14,6 +14,6 @@ class PipConverter : JsonConverter<(int Sys, int Eng, int Wep)>
 
     public override void Write(Utf8JsonWriter writer, (int Sys, int Eng, int Wep) value, JsonSerializerOptions options)
     {
-        JsonSerializer.Serialize(writer, new[] { value.Sys, value.Eng, value.Wep });
+        JsonSerializer.Serialize(writer, new {value.Sys, value.Eng, value.Wep}, options);
     }
 }

ObservatoryFramework/Files/Converters/RepInfConverter.cs

@@ -12,6 +12,6 @@ public class RepInfConverter : JsonConverter<int>
 
     public override void Write(Utf8JsonWriter writer, int value, JsonSerializerOptions options)
     {
-        throw new NotImplementedException();
+        JsonSerializer.Serialize(writer, value, options);
     }
 }

ObservatoryFramework/Files/Converters/StarPosConverter.cs

@@ -17,6 +17,6 @@ public class StarPosConverter : JsonConverter<(double x, double y, double z)>
 
     public override void Write(Utf8JsonWriter writer, (double x, double y, double z) value, JsonSerializerOptions options)
     {
-        throw new NotImplementedException();
+        JsonSerializer.Serialize(writer, value, options);
     }
 }

ObservatoryFramework/Files/Converters/StationServiceConverter.cs

@@ -21,6 +21,6 @@ public class StationServiceConverter : JsonConverter<StationService>
 
     public override void Write(Utf8JsonWriter writer, StationService value, JsonSerializerOptions options)
     {
-        throw new NotImplementedException();
+        JsonSerializer.Serialize(writer, value, options);
     }
 }

ObservatoryFramework/Files/Converters/StringIntConverter.cs

@@ -14,6 +14,6 @@ class StringIntConverter : JsonConverter<int>
 
     public override void Write(Utf8JsonWriter writer, int value, JsonSerializerOptions options)
     {
-        writer.WriteStringValue(value.ToString());
+        JsonSerializer.Serialize(writer, value, options);
     }
 }

ObservatoryFramework/Files/Converters/ThargoidWarRemainingTimeConverter.cs

@@ -7,22 +7,22 @@ class ThargoidWarRemainingTimeConverter : JsonConverter<int>
 {
     public override int Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
     {
-            if (reader.TokenType == JsonTokenType.String)
+        if (reader.TokenType == JsonTokenType.String)
-            {
+        {
-                var value = reader.GetString();
+            var value = reader.GetString();
-                
-                var dayCount = int.TryParse(value.Split(' ')[0], out var days)
-                    ? days
-                    : 0;
 
-                return dayCount;
+            var dayCount = int.TryParse(value.Split(' ')[0], out var days)
-            }
+                ? days
+                : 0;
 
-            return reader.GetInt32();
+            return dayCount;
         }
 
+        return reader.GetInt32();
+    }
+
     public override void Write(Utf8JsonWriter writer, int value, JsonSerializerOptions options)
     {
-            writer.WriteStringValue(value + " Days");
+        writer.WriteStringValue(value + " Days");
-        }
+    }
 }

ObservatoryFramework/Files/Converters/VoucherTypeConverter.cs

@@ -8,18 +8,18 @@ class VoucherTypeConverter : JsonConverter<VoucherType>
 {
     public override VoucherType Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
     {
-            var voucher = reader.GetString();
+        var voucher = reader.GetString();
 
-            if (voucher.Length == 0)
+        if (voucher.Length == 0)
-                voucher = "None";
+            voucher = "None";
 
-            var missionEffect = (VoucherType)Enum.Parse(typeof(VoucherType), voucher, true);
+        var missionEffect = (VoucherType)Enum.Parse(typeof(VoucherType), voucher, true);
 
-            return missionEffect;
+        return missionEffect;
-        }
+    }
 
     public override void Write(Utf8JsonWriter writer, VoucherType value, JsonSerializerOptions options)
     {
-            throw new NotImplementedException();
+        JsonSerializer.Serialize(writer, value, options);
-        }
+    }
 }

ObservatoryFramework/Files/Journal/Travel/FSDJump.cs

@@ -20,15 +20,7 @@ public class FSDJump : JournalBase
     public int BoostUsed { get; init; }
     [JsonConverter(typeof(LegacyFactionConverter<SystemFaction>))]
     public SystemFaction SystemFaction { get; init; }
-    [Obsolete(JournalUtilities.ObsoleteMessage)]
+
-    public string FactionState
-    {
-        get => SystemFaction.FactionState;
-        init
-        {
-                //Stale Data, discard
-            }
-    }
     public string SystemAllegiance { get; init; }
     public string SystemEconomy { get; init; }
     public string SystemEconomy_Localised { get; init; }

ObservatoryFramework/Files/Journal/Travel/Location.cs

@@ -10,16 +10,7 @@ public class Location : JournalBase
     [JsonConverter(typeof(IntBoolFlexConverter))]
     public bool Docked { get; init; }
     public double DistFromStarLS { get; init; }
-
+    
-    [Obsolete(JournalUtilities.ObsoleteMessage)]
-    public string FactionState
-    {
-        get => SystemFaction.FactionState;
-        init
-        {
-                //Stale Data, discard
-            }
-    }
     /// <summary>
     /// Name of the station at which this event occurred.
     /// </summary>

Pulsar/Features/FileHandlerService.cs

@@ -89,6 +89,6 @@ public class FileHandlerService(
         }
             
         logger.LogInformation("Handling file {FileName} with Type {Type}", fileName, handler.GetType().ToString());
-        Task.Run(() => handler.HandleFile(path));
+        await handler.HandleFile(path);
     }
 }

Pulsar/Features/FileWatcherService.cs

@@ -3,44 +3,67 @@ namespace Pulsar.Features;
 using System.Collections.Concurrent;
 using Microsoft.Extensions.FileProviders;
 
-public class FileWatcherService(IOptions<PulsarConfiguration> options, IFileHandlerService fileHandlerService) : IHostedService
+public class FileWatcherService(IOptions<PulsarConfiguration> options, IFileHandlerService fileHandlerService)
+    : IHostedService
 {
     private PhysicalFileProvider watcher = null!;
-    
+
     public Task StartAsync(CancellationToken cancellationToken)
     {
         if (!Directory.Exists(options.Value.JournalDirectory))
         {
             throw new Exception($"Directory {options.Value.JournalDirectory} does not exist.");
         }
-        
+
         watcher = new PhysicalFileProvider(options.Value.JournalDirectory);
         Watch();
 
+        // read the journal directory to get the initial files
+#if DEBUG
+        Task.Run(() =>
+        {
+            Thread.Sleep(TimeSpan.FromSeconds(10));
+            HandleFileChanged();
+        }, cancellationToken);
+#else
+        HandleFileChanged();
+#endif
+
+
         return Task.CompletedTask;
     }
-    
+
     ConcurrentDictionary<string, DateTimeOffset> FileDates = new();
 
-    private void HandleFileChanged(object? sender)
+    private void HandleFileChanged(object? sender = null)
     {
         foreach (var file in watcher.GetDirectoryContents(""))
         {
-            if (file.IsDirectory || !file.Name.EndsWith(".json") && !(file.Name.StartsWith(FileHandlerService.JournalLogFileNameStart) && file.Name.EndsWith(FileHandlerService.JournalLogFileNameEnd)))
+            if (file.IsDirectory || !file.Name.EndsWith(".json") &&
+                !(file.Name.StartsWith(FileHandlerService.JournalLogFileNameStart) &&
+                  file.Name.EndsWith(FileHandlerService.JournalLogFileNameEnd)))
             {
                 continue;
             }
-            
+
-            var existing = FileDates.GetOrAdd(file.PhysicalPath, file.LastModified);
+            FileDates.AddOrUpdate(file.PhysicalPath, _ =>
-            
-            if (existing != file.LastModified)
             {
-                fileHandlerService.HandleFile(file.PhysicalPath);
+                Task.Run(() => fileHandlerService.HandleFile(file.PhysicalPath));
-            }
+                return file.LastModified;
+            }, (_, existing) =>
+            {
+                if (existing != file.LastModified)
+                {
+                    Task.Run(() => fileHandlerService.HandleFile(file.PhysicalPath));
+                }
+
+                return file.LastModified;
+            });
         }
+
         Watch();
     }
-    
+
     private void Watch()
     {
         watcher.Watch("*.*").RegisterChangeCallback(HandleFileChanged, null);

Pulsar/Features/Journal/JournalService.cs

@@ -1,12 +1,12 @@
 namespace Pulsar.Features.Journal;
 
+using System.Collections.Concurrent;
 using System.Text.RegularExpressions;
 using Observatory.Framework.Files.Journal;
 
 public interface IJournalService : IJournalHandler<List<JournalBase>>;
 
-public class JournalService
+public class JournalService(
-(
     ILogger<JournalService> logger,
     IOptions<PulsarConfiguration> options,
     IEventHubContext hub,
@@ -14,74 +14,52 @@ public class JournalService
 ) : IJournalService
 {
     public string FileName => FileHandlerService.JournalLogFileName;
+
+    static ConcurrentBag<JournalBase> _journals = new();
     
-    public Task HandleFile(string filePath) => HandleFile(filePath, CancellationToken.None);
+    static DateTimeOffset notBefore = DateTimeOffset.UtcNow.AddHours(-1);
-    public async Task HandleFile(string filePath, CancellationToken token)
+
+    public async Task HandleFile(string filePath)
     {
         if (!FileHelper.ValidateFile(filePath))
         {
             return;
         }
 
-        var file = await File.ReadAllLinesAsync(filePath, Encoding.UTF8, token);
+        var file = await File.ReadAllLinesAsync(filePath, Encoding.UTF8);
-        var journals = file.Select(line => JsonSerializer.Deserialize<JournalBase>(line)).ToList();
-
-        
         var newJournals = new List<JournalBase>();
-        var notBefore = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(6));
+        var select = file.AsParallel().Select(line => JsonSerializer.Deserialize<JournalBase>(line,
-        foreach (var journal in journals)
+            new JsonSerializerOptions
+            {
+                PropertyNameCaseInsensitive = true,
+                Converters = { new JournalConverter(logger) }
+            }));
+        
+        foreach (var journal in select)
         {
-            if (context.Journals.Any(j => j.Timestamp == journal.Timestamp && j.Event == journal.Event))
+            if (_journals.Any(j => j.Timestamp == journal.Timestamp && j.Event == journal.Event))
+            {
+                continue;
+            }
+            
+            if (journal.Timestamp < notBefore)
             {
                 continue;
             }
 
-            context.Journals.Add(journal);
+            _journals.Add(journal);
-            
+            newJournals.Add(journal);
-            if (journal.Timestamp > notBefore)
-            {
-                newJournals.Add(journal);
-            }
         }
 
-        await hub.Clients.All.JournalUpdated(newJournals);
+        if (newJournals.Any())
+        {
+            await hub.Clients.All.JournalUpdated(newJournals);
+        }
     }
 
     public async Task<List<JournalBase>> Get()
     {
-        var folder = new DirectoryInfo(options.Value.JournalDirectory);
+        await hub.Clients.All.JournalUpdated(_journals.ToList());
-        var regex = new Regex(FileHandlerService.JournalLogFileNameRegEx);
+        return _journals.ToList();
-        
-        if (!folder.Exists)
-        {
-            logger.LogWarning("Journal directory {JournalDirectory} does not exist", folder.FullName);
-            return [];
-        }
-        
-        var dataFileName = folder.GetFiles().FirstOrDefault(f => regex.IsMatch(f.Name))?.FullName;
-        
-        if (!FileHelper.ValidateFile(dataFileName))
-        {
-            return [];
-        }
-
-        // Seems each entry is a new line. Not sure if this can be relied on?
-        var logs = File.ReadAllLines(dataFileName);
-
-        var journals = new List<JournalBase>();
-        foreach (var log in logs)
-        {
-            // var info = JournalReader.ObservatoryDeserializer<JournalBase>(log);
-            var info = JsonSerializer.Deserialize<JournalBase>(log);
-            if (info != null)
-            {
-                journals.Add(info);
-            }
-        }
-
-        if (journals.Count > 0) return journals;
-
-        logger.LogWarning("Failed to deserialize module info file {file}", dataFileName);
-        return [];
     }
-}
+}

Pulsar/Features/Status/StatusService.cs

@@ -13,31 +13,6 @@ public class StatusService
 {
     public string FileName => FileHandlerService.StatusFileName;
 
-    public bool ValidateFile(string filePath)
-    {
-        if (!File.Exists(filePath))
-        {
-            logger.LogWarning("Journal file {JournalFile} does not exist", filePath);
-            return false;
-        }
-
-        var fileInfo = new FileInfo(filePath);
-
-        if (!string.Equals(fileInfo.Name, FileName, StringComparison.InvariantCultureIgnoreCase))
-        {
-            logger.LogWarning("Journal file {name} is not valid");
-            return false;
-        }
-
-        if (fileInfo.Length == 0)
-        {
-            logger.LogWarning("Journal file {name} is empty", filePath);
-            return false;
-        }
-
-        return true;
-    }
-
     public async Task HandleFile(string filePath)
     {
         if (!FileHelper.ValidateFile(filePath))
@@ -46,6 +21,13 @@ public class StatusService
         }
 
         var file = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+        
+        if (file.Length < 2)
+        {
+            logger.LogWarning("File {FilePath} is empty", filePath);
+            return;
+        }
+            
         var status = await JsonSerializer.DeserializeAsync<Status>(file);
 
         if (status == null)