quartznet/pulsar
2
0
Fork 0
mirror of https://github.com/9ParsonsB/Pulsar.git synced 2025-07-01 08:23:42 -04:00
Code Actions Activity

Journals Now processed in own thread

Browse Source 
Some invalid journal data is now handled
Journals now use polymorphic deserialization
Added Event names to all journal events
Remove unused controllers
This commit is contained in:
Ben Parsons
2024-05-24 17:57:10 +10:00
parent ae848e036d
commit efd0b3e0c0
297 changed files with 827 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
Pulsar/Features/Cargo/CargoController.cs
View File

@ -1,13 +0,0 @@
using Observatory.Framework.Files;
namespace Pulsar.Features.Cargo;
[ApiController]
[Route("api/[controller]")]
public class CargoController(ICargoService cargoService) : ControllerBase
{
public async Task<ActionResult<CargoFile>> Get()
{
return Ok(await cargoService.Get());
}
}

4
Pulsar/Features/Cargo/CargoService.cs
View File

@ -8,7 +8,7 @@ public class CargoService(IOptions<PulsarConfiguration> options, ILogger<CargoSe
{
public string FileName => "Cargo.json";
public async Task HandleFile(string filePath)
public async Task HandleFile(string filePath, CancellationToken token = new())
{
if (!FileHelper.ValidateFile(filePath))
{
@ -16,7 +16,7 @@ public class CargoService(IOptions<PulsarConfiguration> options, ILogger<CargoSe
}
var file = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
var moduleInfo = await JsonSerializer.DeserializeAsync<CargoFile>(file);
var moduleInfo = await JsonSerializer.DeserializeAsync<CargoFile>(file, cancellationToken: token);
if (moduleInfo == null)
{

8
Pulsar/Features/EventsHub.cs
View File

@ -6,6 +6,12 @@ using Observatory.Framework.Files.Journal.Odyssey;
public class EventsHub : Hub<IEventsHub>
{
public async Task Status([FromServices] IStatusService statusService)
{
var status = await statusService.Get();
await Clients.Caller.StatusUpdated(status);
}
public async Task StatusUpdated(Observatory.Framework.Files.Status status) => await Clients.All.StatusUpdated(status);
public async Task OutfittingUpdated(OutfittingFile outfitting) => await Clients.All.OutfittingUpdated(outfitting);
@ -21,7 +27,7 @@ public class EventsHub : Hub<IEventsHub>
public async Task ModuleInfoUpdated(ModuleInfoFile moduleInfo) => await Clients.All.ModuleInfoUpdated(moduleInfo);
public async Task FleetCarrierUpdated(FCMaterialsFile fleetCarrier) => await Clients.All.FleetCarrierUpdated(fleetCarrier);
public async Task CargoUpdated(CargoFile cargo) => await Clients.All.CargoUpdated(cargo);
public async Task BackpackUpdated(BackpackFile backpack) => await Clients.All.BackpackUpdated(backpack);

10
Pulsar/Features/FileHandlerService.cs
View File

@ -1,3 +1,4 @@
using System.Collections.Concurrent;
using Observatory.Framework.Files;
using Observatory.Framework.Files.Journal;
using Observatory.Framework.Files.Journal.Odyssey;
@ -5,7 +6,7 @@ namespace Pulsar.Features;
public interface IFileHandler
{
Task HandleFile(string path);
Task HandleFile(string path, CancellationToken token = new());
}
public interface IFileHandlerService : IFileHandler;
@ -60,8 +61,9 @@ public class FileHandlerService(
{ OutfittingFileName, typeof(IJournalHandler<OutfittingFile>) },
{ JournalLogFileNameStart, typeof(IJournalHandler<List<JournalBase>>) }
};
public async Task HandleFile(string path)
public async Task HandleFile(string path, CancellationToken token = new())
{
var fileInfo = new FileInfo(path);
var fileName = fileInfo.Name;
@ -78,7 +80,7 @@ public class FileHandlerService(
if (!Handlers.TryGetValue(match, out var type))
{
logger.LogInformation("File {FileName} was not handled", fileName);
logger.LogWarning("File {FileName} was not handled", fileName);
return;
}
@ -89,6 +91,6 @@ public class FileHandlerService(
}
logger.LogInformation("Handling file {FileName} with Type {Type}", fileName, handler.GetType().ToString());
await handler.HandleFile(path);
await handler.HandleFile(path, token);
}
}

31
Pulsar/Features/FileWatcherService.cs
View File

@ -16,17 +16,17 @@ public class FileWatcherService(IOptions<PulsarConfiguration> options, IFileHand
}
watcher = new PhysicalFileProvider(options.Value.JournalDirectory);
Watch();
Watch(cancellationToken);
// read the journal directory to get the initial files
#if DEBUG
Task.Run(() =>
{
Thread.Sleep(TimeSpan.FromSeconds(10));
HandleFileChanged();
Thread.Sleep(TimeSpan.FromSeconds(2));
HandleFileChanged(cancellationToken);
}, cancellationToken);
#else
HandleFileChanged();
HandleFileChanged(cancellationToken);
#endif
@ -35,38 +35,47 @@ public class FileWatcherService(IOptions<PulsarConfiguration> options, IFileHand
ConcurrentDictionary<string, DateTimeOffset> FileDates = new();
private void HandleFileChanged(object? sender = null)
private void HandleFileChanged(CancellationToken token = new())
{
var tasks = new List<Task>();
foreach (var file in watcher.GetDirectoryContents(""))
{
if (file.IsDirectory || !file.Name.EndsWith(".json") &&
!(file.Name.StartsWith(FileHandlerService.JournalLogFileNameStart) &&
file.Name.EndsWith(FileHandlerService.JournalLogFileNameEnd)))
{
continue;
return;
}
FileDates.AddOrUpdate(file.PhysicalPath, _ =>
{
Task.Run(() => fileHandlerService.HandleFile(file.PhysicalPath));
tasks.Add(Task.Run(() => fileHandlerService.HandleFile(file.PhysicalPath, token), token));
return file.LastModified;
}, (_, existing) =>
{
if (existing != file.LastModified)
{
Task.Run(() => fileHandlerService.HandleFile(file.PhysicalPath));
tasks.Add(Task.Run(() => fileHandlerService.HandleFile(file.PhysicalPath, token), token));
}
return file.LastModified;
});
}
Watch();
Watch(token);
Task.WaitAll(tasks.ToArray(), token);
}
private void Watch()
private void Watch(CancellationToken token)
{
watcher.Watch("*.*").RegisterChangeCallback(HandleFileChanged, null);
void Handle(object? _)
{
HandleFileChanged(token);
}
watcher.Watch("*.*").RegisterChangeCallback(Handle, null);
}
public Task StopAsync(CancellationToken cancellationToken)

109
Pulsar/Features/Journal/JournalProcessor.cs Normal file
View File

@ -0,0 +1,109 @@
namespace Pulsar.Features.Journal;
using Observatory.Framework;
using Observatory.Framework.Files.Journal;
public class JournalProcessor(
ILogger<JournalProcessor> logger,
IJournalService journalService,
IEventHubContext hub) : IJournalProcessor, IHostedService, IDisposable
{
private CancellationTokenSource tokenSource = new();
readonly JsonSerializerOptions options = new()
{
PropertyNameCaseInsensitive = true,
AllowOutOfOrderMetadataProperties = true,
Converters = { new JournalInvalidDoubleConverter(), new JournalInvalidFloatConverter() },
NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals,
};
public async Task HandleFileInner(string filePath, CancellationToken token = new())
{
logger.LogInformation("Processing journal file: '{File}'", filePath);
var file = await File.ReadAllBytesAsync(filePath, token);
var lines = file.Split(Encoding.UTF8.GetBytes(Environment.NewLine)).ToList();
var newJournals = new List<JournalBase>();
//await Parallel.ForEachAsync(lines, new ParallelOptions() { MaxDegreeOfParallelism = 32, TaskScheduler = TaskScheduler.Default, CancellationToken = token}, (line, _) =>
for (var index = 0; index < lines.Count; index++)
{
var line = lines[index];
if (line.Count == 0)
{
// return ValueTask.CompletedTask;
continue;
}
if (line.Contains("\"RotationPeriod\":inf"u8.ToArray()))
{
var newLine = line.Replace("\"RotationPeriod\":inf"u8, "\"RotationPeriod\":0"u8);
line = newLine;
}
try
{
var journal = JsonSerializer.Deserialize<JournalBase>(new ReadOnlySpan<byte>(line.ToArray()), options);
if (journal == null)
{
//return ValueTask.CompletedTask;
continue;
}
newJournals.Add(journal);
}
catch (JsonException ex)
{
logger.LogError(ex, "Error deserializing journal file: '{File}', line: {Line}", filePath, line);
}
//return ValueTask.CompletedTask;
}
if (newJournals.Any())
{
await hub.Clients.All.JournalUpdated(newJournals);
}
}
public Task StartAsync(CancellationToken cancellationToken)
{
tokenSource.Dispose();
tokenSource = new();
ProcessQueue();
return Task.CompletedTask;
}
private void ProcessQueue()
{
Task.Run(async () =>
{
while (!tokenSource.Token.IsCancellationRequested)
{
if (journalService.TryDequeue(out var file))
{
await HandleFileInner(file, tokenSource.Token);
}
else
{
await Task.Delay(1000);
}
}
}, tokenSource.Token);
}
public Task StopAsync(CancellationToken cancellationToken)
{
tokenSource?.Cancel();
return Task.CompletedTask;
}
public void Dispose()
{
tokenSource?.Dispose();
}
}
public interface IJournalProcessor
{
}

73
Pulsar/Features/Journal/JournalService.cs
View File

@ -1,77 +1,42 @@
namespace Pulsar.Features.Journal;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Observatory.Framework.Files.Journal;
public interface IJournalService : IJournalHandler<List<JournalBase>>;
public interface IJournalService : IJournalHandler<List<JournalBase>>
{
public bool TryDequeue(out string filePath);
}
public class JournalService(
ILogger<JournalService> logger,
IOptions<PulsarConfiguration> options,
IEventHubContext hub,
PulsarContext context,
IServiceProvider serviceProvider
ILogger<JournalService> logger
) : IJournalService
{
public string FileName => FileHandlerService.JournalLogFileName;
static ConcurrentBag<JournalBase> _journals = new();
private readonly ConcurrentQueue<string> JournalFileQueue = new();
static DateTimeOffset notBefore = DateTimeOffset.UtcNow.AddHours(-1);
readonly JsonSerializerOptions options = new()
public void EnqueueFile(string filePath)
{
PropertyNameCaseInsensitive = true,
AllowOutOfOrderMetadataProperties = true,
// Converters = { ActivatorUtilities.CreateInstance<JournalJsonConverter>(serviceProvider) }
};
JournalFileQueue.Enqueue(filePath);
}
public bool TryDequeue(out string filePath)
{
return JournalFileQueue.TryDequeue(out filePath);
}
public async Task HandleFile(string filePath)
public Task HandleFile(string filePath, CancellationToken token = new())
{
if (!FileHelper.ValidateFile(filePath))
{
return;
return Task.CompletedTask;
}
var file = await File.ReadAllLinesAsync(filePath, Encoding.UTF8);
var newJournals = new List<JournalBase>();
await Parallel.ForEachAsync(file, (line, _) =>
{
if (string.IsNullOrWhiteSpace(line))
{
return ValueTask.CompletedTask;
}
var journal = JsonSerializer.Deserialize<JournalBase>(line, options);
if (journal == null)
{
return ValueTask.CompletedTask;
}
if (_journals.Any(j => j.Timestamp == journal.Timestamp && j.GetType() == journal.GetType()))
{
return ValueTask.CompletedTask;
}
_journals.Add(journal);
if (journal.Timestamp < notBefore)
{
return ValueTask.CompletedTask;
}
newJournals.Add(journal);
return ValueTask.CompletedTask;
});
if (newJournals.Any())
{
await hub.Clients.All.JournalUpdated(newJournals);
}
EnqueueFile(filePath);
return Task.CompletedTask;
}
public async Task<List<JournalBase>> Get()
{
return [];

12
Pulsar/Features/ModulesInfo/ModulesInfoController.cs
View File

@ -1,12 +0,0 @@
namespace Pulsar.Features.ModulesInfo;
[ApiController]
[Route("api/[controller]")]
public class ModulesInfoController(IModulesInfoService modulesInfo) : ControllerBase
{
[HttpGet]
public async Task<IActionResult> Get()
{
return Ok(await modulesInfo.Get());
}
}

4
Pulsar/Features/ModulesInfo/ModulesInfoService.cs
View File

@ -11,7 +11,7 @@ public class ModulesInfoService(
{
public string FileName => FileHandlerService.ModulesInfoFileName;
public async Task HandleFile(string filePath)
public async Task HandleFile(string filePath, CancellationToken token = new())
{
if (!FileHelper.ValidateFile(filePath))
{
@ -19,7 +19,7 @@ public class ModulesInfoService(
}
var file = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
var moduleInfo = await JsonSerializer.DeserializeAsync<ModuleInfoFile>(file);
var moduleInfo = await JsonSerializer.DeserializeAsync<ModuleInfoFile>(file, cancellationToken: token);
if (moduleInfo == null)
{

12
Pulsar/Features/ShipLocker/ShipLockerController.cs
View File

@ -1,12 +0,0 @@
namespace Pulsar.Features.ShipLocker;
[ApiController]
[Route("api/[controller]")]
public class ShipLockerController(IShipLockerService shipLockerService) : ControllerBase
{
[HttpGet]
public async Task<IActionResult> Get()
{
return Ok(await shipLockerService.Get());
}
}

4
Pulsar/Features/ShipLocker/ShipLockerService.cs
View File

@ -27,7 +27,7 @@ public class ShipLockerService(ILogger<ShipLockerService> logger, IOptions<Pulsa
return new ShipLockerMaterials();
}
public async Task HandleFile(string filePath)
public async Task HandleFile(string filePath, CancellationToken token = new())
{
if (!FileHelper.ValidateFile(filePath))
{
@ -35,7 +35,7 @@ public class ShipLockerService(ILogger<ShipLockerService> logger, IOptions<Pulsa
}
var file = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
var shipLocker = await JsonSerializer.DeserializeAsync<ShipLockerMaterials>(file);
var shipLocker = await JsonSerializer.DeserializeAsync<ShipLockerMaterials>(file, cancellationToken: token);
if (shipLocker == null)
{

4
Pulsar/Features/Shipyard/ShipyardService.cs
View File

@ -25,7 +25,7 @@ public class ShipyardService(ILogger<ShipyardService> logger, IOptions<PulsarCon
return new ShipyardFile();
}
public async Task HandleFile(string path)
public async Task HandleFile(string path, CancellationToken token = new())
{
if (!FileHelper.ValidateFile(path))
{
@ -33,7 +33,7 @@ public class ShipyardService(ILogger<ShipyardService> logger, IOptions<PulsarCon
}
var file = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
var shipyard = await JsonSerializer.DeserializeAsync<ShipyardFile>(file);
var shipyard = await JsonSerializer.DeserializeAsync<ShipyardFile>(file, cancellationToken: token);
if (shipyard == null)
{

12
Pulsar/Features/Status/StatusController.cs
View File

@ -1,12 +0,0 @@
namespace Pulsar.Features.Status;
[ApiController]
[Route("api/[controller]")]
public class StatusController(IStatusService status) : ControllerBase
{
[HttpGet]
public async Task<IActionResult> Get()
{
return Ok(await status.Get());
}
}

21
Pulsar/Features/Status/StatusService.cs
View File

@ -1,3 +1,6 @@
using Microsoft.EntityFrameworkCore;
using Observatory.Framework.Files.Journal.Startup;
namespace Pulsar.Features.Status;
using Observatory.Framework.Files;
@ -8,12 +11,13 @@ public class StatusService
(
ILogger<StatusService> logger,
IOptions<PulsarConfiguration> options,
IEventHubContext hub
IEventHubContext hub,
PulsarContext context
) : IStatusService
{
public string FileName => FileHandlerService.StatusFileName;
public async Task HandleFile(string filePath)
public async Task HandleFile(string filePath, CancellationToken token = new())
{
if (!FileHelper.ValidateFile(filePath))
{
@ -28,7 +32,7 @@ public class StatusService
return;
}
var status = await JsonSerializer.DeserializeAsync<Status>(file);
var status = await JsonSerializer.DeserializeAsync<Status>(file, cancellationToken: token);
if (status == null)
{
@ -45,14 +49,19 @@ public class StatusService
if (!FileHelper.ValidateFile(statusFile))
{
return new Status();
return new ();
}
await using var file = File.Open(statusFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
var status = await JsonSerializer.DeserializeAsync<Status>(file);
if (status != null) return status;
if (status != null)
{
return status;
}
logger.LogWarning("Failed to deserialize status file {StatusFile}", statusFile);
return new Status();
return new ();
}
}