feat: Support persisting SQLite DB, and data refresher

.github/workflows/openactive-test-suite.yml

@@ -57,7 +57,7 @@ jobs:
  # output
  - name: Run OpenActive.FakeDatabase.NET.Tests
  run: dotnet test ./Fakes/OpenActive.FakeDatabase.NET.Tests/OpenActive.FakeDatabase.NET.Tests.csproj --configuration Release --no-build --verbosity normal
-
+
  core:
  # Specifies that this job depends on the successful completion of two other jobs: "test-server" and "test-fake-database"
  needs:
@@ -110,6 +110,7 @@ jobs:
 
  # runs `dotnet restore` to install the dependencies for the "OpenActive.Server.NET" project. It is conditional and
  # depends on the "profile" value not being 'no-auth' or 'single-seller'
+ # LW: Why is it conditional?
  - name: Install OpenActive.Server.NET dependencies
  if: ${{ matrix.profile != 'no-auth' && matrix.profile != 'single-seller' }}
  run: dotnet restore ./server/

.gitignore

@@ -336,3 +336,6 @@ ASALocalRun/
 
 # Fake database
 *fakedatabase.db
+
+# Output path for app publishing
+/web-app-package/

Examples/BookingSystem.AspNetCore.IdentityServer/Startup.cs

@@ -8,6 +8,7 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Configuration;
 using OpenActive.FakeDatabase.NET;
+using Microsoft.Extensions.Logging;
 
 namespace IdentityServer
 {
@@ -27,7 +28,11 @@ public Startup(IWebHostEnvironment environment, IConfiguration configuration)
  public void ConfigureServices(IServiceCollection services)
  {
  services.AddTransient<IClientStore, ClientStore>();
- services.AddSingleton(x => new FakeBookingSystem(AppSettings.FeatureFlags.FacilityUseHasSlots));
+ services.AddSingleton(x => new FakeBookingSystem
+ (
+ AppSettings.FeatureFlags.FacilityUseHasSlots,
+ x.GetRequiredService<ILogger<FakeBookingSystem>>()
+ ));
 
  var builder = services.AddIdentityServer(options =>
  {

Examples/BookingSystem.AspNetCore/BackgroundServices/FakeDataRefresherService.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using OpenActive.FakeDatabase.NET;
+using BookingSystem.AspNetCore.Services;
+
+
+namespace BookingSystem
+{
+ /// <summary>
+ /// A background task which periodically refreshes the data in the
+ /// FakeBookingSystem. This means that past data is deleted and new copies
+ /// are created in the future.
+ ///
+ /// More information on background tasks here:
+ /// https://docs.microsoft.com/en-us/dotnet/architecture/microservices/multi-container-microservice-net-applications/background-tasks-with-ihostedservice#implementing-ihostedservice-with-a-custom-hosted-service-class-deriving-from-the-backgroundservice-base-class
+ /// </summary>
+ public class FakeDataRefresherService : BackgroundService
+ {
+ private readonly ILogger<FakeDataRefresherService> _logger;
+ private readonly AppSettings _settings;
+ private readonly FakeBookingSystem _bookingSystem;
+ private readonly DataRefresherStatusService _statusService;
+
+ public FakeDataRefresherService(
+ AppSettings settings, 
+ ILogger<FakeDataRefresherService> logger,
+ FakeBookingSystem bookingSystem,
+ DataRefresherStatusService statusService)
+ {
+ _settings = settings;
+ _logger = logger;
+ _bookingSystem = bookingSystem;
+ _statusService = statusService;
+
+ // Indicate that the refresher service is configured to run
+ _statusService.SetRefresherConfigured(true);
+ }
+
+ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+ {
+ var interval = TimeSpan.FromHours(_settings.DataRefresherIntervalHours);
+
+ stoppingToken.Register(() =>
+ _logger.LogInformation($"FakeDataRefresherService background task is stopping."));
+
+ while (!stoppingToken.IsCancellationRequested)
+ {
+ _logger.LogInformation($"FakeDataRefresherService is starting..");
+ var (numDeletedOccurrences, numDeletedSlots) = await _bookingSystem
+ .Database
+ .HardDeleteOldSoftDeletedOccurrencesAndSlots();
+ _logger.LogInformation($"FakeDataRefresherService hard deleted {numDeletedOccurrences} occurrences and {numDeletedSlots} slots that were previously old and soft-deleted.");
+
+ var (numRefreshedOccurrences, numRefreshedSlots) = await _bookingSystem
+ .Database
+ .SoftDeletePastOpportunitiesAndInsertNewAtEdgeOfWindow();
+ _logger.LogInformation($"FakeDataRefresherService, for {numRefreshedOccurrences} old occurrences and {numRefreshedSlots} old slots, inserted new copies into the future and soft-deleted the old ones.");
+
+ _logger.LogInformation($"FakeDataRefresherService is finished");
+
+ // Signal that a cycle has completed
+ _statusService.SignalCycleCompletion();
+
+ await Task.Delay(interval, stoppingToken);
+ }
+ }
+ }
+}

Examples/BookingSystem.AspNetCore/Controllers/InitWaitController.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Threading.Tasks;
+using BookingSystem.AspNetCore.Services;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+
+/// <summary>
+/// Endpoints which are used to wait for components within
+/// BookingSystem.AspNetCore to be initialized.
+/// </summary>
+namespace BookingSystem.AspNetCore.Controllers
+{
+ [ApiController]
+ [Route("init-wait")]
+ public class InitWaitController : ControllerBase
+ {
+ private readonly DataRefresherStatusService _statusService;
+ private readonly ILogger<InitWaitController> _logger;
+ private static readonly TimeSpan _defaultTimeout = TimeSpan.FromMinutes(5);
+
+ public InitWaitController(
+ DataRefresherStatusService statusService,
+ ILogger<InitWaitController> logger)
+ {
+ _statusService = statusService;
+ _logger = logger;
+ }
+
+ /// <summary>
+ /// Wait for the data refresher to complete its first cycle.
+ ///
+ /// This makes it possible to write scripts (for CI) which don't start
+ /// until the data refresher has completed at least one cycle.
+ ///
+ /// - Returns 204 when the data refresher has completed its first cycle.
+ /// - Returns 503 if the data refresher is not configured to run.
+ /// - Returns 504 if the data refresher fails to complete a cycle within
+ /// the default timeout.
+ /// </summary>
+ [HttpGet("data-refresher")]
+ public async Task<IActionResult> WaitForDataRefresher()
+ {
+ _logger.LogDebug("Received request to wait for data refresher completion");
+
+ // Check if the data refresher is configured to run
+ if (!_statusService.IsRefresherConfigured())
+ {
+ _logger.LogWarning("Data refresher is not configured to run");
+ return StatusCode(503, "Data refresher service is not configured to run");
+ }
+
+ // If it has already completed a cycle, return immediately
+ if (_statusService.HasCompletedCycle())
+ {
+ _logger.LogDebug("Data refresher has already completed a cycle");
+ return NoContent();
+ }
+
+ _logger.LogDebug("Waiting for data refresher to complete a cycle...");
+
+ // Wait for the cycle to complete, with a timeout
+ await _statusService.WaitForCycleCompletion(_defaultTimeout);
+
+ if (_statusService.HasCompletedCycle())
+ {
+ _logger.LogDebug("Data refresher completed a cycle, returning 204");
+ return NoContent();
+ }
+ else
+ {
+ _logger.LogWarning("Timed out waiting for data refresher to complete a cycle");
+ return StatusCode(504, "Timed out waiting for data refresher to complete a cycle");
+ }
+ }
+ }
+} 

Examples/BookingSystem.AspNetCore/Feeds/FacilitiesFeeds.cs

@@ -1,5 +1,5 @@
-using Bogus;
-using BookingSystem.AspNetCore.Helpers;
+using Bogus;
+using BookingSystem.AspNetCore.Helpers;
 using OpenActive.DatasetSite.NET;
 using OpenActive.FakeDatabase.NET;
 using OpenActive.NET;
@@ -26,6 +26,7 @@ public AcmeFacilityUseRpdeGenerator(AppSettings appSettings, FakeBookingSystem f
  this._fakeBookingSystem = fakeBookingSystem;
  }
 
+ // TODO this method should use async queries so as not to block the main thread
  protected override async Task<List<RpdeItem<FacilityUse>>> GetRpdeItems(long? afterTimestamp, long? afterId)
  {
  var facilityTypeId = Environment.GetEnvironmentVariable("FACILITY_TYPE_ID") ?? "https://openactive.io/facility-types#a1f82b7a-1258-4d9a-8dc5-bfc2ae961651";
@@ -204,6 +205,7 @@ public AcmeFacilityUseSlotRpdeGenerator(AppSettings appSettings, FakeBookingSyst
  this._fakeBookingSystem = fakeBookingSystem;
  }
 
+ // TODO this method should use async queries so as not to block the main thread
  protected override async Task<List<RpdeItem<Slot>>> GetRpdeItems(long? afterTimestamp, long? afterId)
  {
  using (var db = _fakeBookingSystem.Database.Mem.Database.Open())

Examples/BookingSystem.AspNetCore/README.md

@@ -33,6 +33,15 @@ ASPNETCORE_ENVIRONMENT=no-auth dotnet run --no-launch-profile --project ./Bookin
 
 The above example starts the BookingSystem.AspNetCore in `no-auth` mode.
 
+## Env Vars
+
+BookingSystem.AspNetCore uses the following environment variables (note: this list is not exhaustive):
+
+- `PERIODICALLY_REFRESH_DATA`: (optional - default `false`) If set to `true`,
+ the database will be periodically refreshed, deleting past data and replacing
+ it with future data.
+- `IS_LOREM_FITSUM_MODE`: (optional - default `false`) If set to `true`, data is created in ["Lorem Fitsum" mode](#lorem-fitsum-mode).
+
 ## BookingSystem.AspNetCore Data Generation
 
 BookingSystem.AspNetCore has three main uses that make it very important in the OpenActive ecosystem:
@@ -64,6 +73,6 @@ In the CLI this can be done by running the following command for example:
 IS_LOREM_FITSUM_MODE=true dotnet run --no-launch-profile --project ./BookingSystem.AspNetCore.csproj --configuration Release --no-build
 ```
 
-### Golden Records
+#### Golden Records
 Golden records are randomly generated records that have maximally enriched properties in the generated data. For example where a record might have one image normally, a golden record will have four.
 

Examples/BookingSystem.AspNetCore/Services/DataRefresherStatusService.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BookingSystem.AspNetCore.Services
+{
+ /// <summary>
+ /// A service which tracks the status of the data refresher, including
+ /// whether it is configured to run and whether a cycle has completed.
+ /// </summary>
+ public class DataRefresherStatusService
+ {
+ private readonly SemaphoreSlim _completionSemaphore = new SemaphoreSlim(0, 1);
+ private bool _isRefresherConfigured = false;
+ private bool _hasCompletedCycle = false;
+
+ public void SetRefresherConfigured(bool isConfigured)
+ {
+ _isRefresherConfigured = isConfigured;
+ }
+
+ public bool IsRefresherConfigured()
+ {
+ return _isRefresherConfigured;
+ }
+
+ public void SignalCycleCompletion()
+ {
+ _hasCompletedCycle = true;
+
+ // Release the semaphore if someone is waiting on it
+ if (_completionSemaphore.CurrentCount == 0)
+ {
+ _completionSemaphore.Release();
+ }
+ }
+
+ /// <summary>
+ /// Has the data refresher completed a cycle?
+ ///
+ /// This makes it possible to write scripts (for CI) which don't start
+ /// until the data refresher has completed at least one cycle.
+ /// </summary>
+ public bool HasCompletedCycle()
+ {
+ return _hasCompletedCycle;
+ }
+
+ public async Task WaitForCycleCompletion(TimeSpan timeout)
+ {
+ if (_hasCompletedCycle)
+ {
+ return;
+ }
+
+ await _completionSemaphore.WaitAsync(timeout);
+ }
+ }
+} 

Examples/BookingSystem.AspNetCore/Settings/AppSettings.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace BookingSystem
 {
  public class AppSettings
@@ -6,6 +8,7 @@ public class AppSettings
  public string OpenIdIssuerUrl { get; set; }
  public FeatureSettings FeatureFlags { get; set; }
  public PaymentSettings Payment { get; set; }
+ public int DataRefresherIntervalHours { get; set; } = 6;
  }
 
  /**