[Migration Engine Part 3] Implement the RavenDB to SQL migration machinery

README.md

@@ -35,7 +35,7 @@ the `launchSettings.json` file of each instance. When started in setup mode, the
 
 ## Migrating from RavenDB to SQL Server or PostgreSQL
 
-See [Migrating from RavenDB to SQL Server or PostgreSQL](docs/migration/ravendb-to-sql-migration-instructions.md).
+- [Migrate from RavenDB to SQL Server or PostgreSQL](docs/migration/ravendb-to-sql-migration-instructions.md): what an operator can run today, and what is not built yet.
 
 ## Secrets
 

src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj

@@ -12,6 +12,9 @@
   <ItemGroup>
     <ProjectReference Include="..\ServiceControl.AcceptanceTesting\ServiceControl.AcceptanceTesting.csproj" />
     <ProjectReference Include="..\ServiceControl.Persistence.EFCore.PostgreSql\ServiceControl.Persistence.EFCore.PostgreSql.csproj" />
+    <!-- Built so its manifest is where DevelopmentPersistenceLocations looks; not copied, or its persistence.manifest would collide with this project's own. -->
+    <ProjectReference Include="..\ServiceControl.Persistence.RavenDB\ServiceControl.Persistence.RavenDB.csproj" ReferenceOutputAssembly="false" Private="false" />
+    <ProjectReference Include="..\ServiceControl.RavenDB\ServiceControl.RavenDB.csproj" />
     <ProjectReference Include="..\ServiceControl.Transports.Learning\ServiceControl.Transports.Learning.csproj" />
     <ProjectReference Include="..\ServiceControl\ServiceControl.csproj" />
     <ProjectReference Include="..\TestHelper\TestHelper.csproj" />
@@ -20,6 +23,7 @@
   <ItemGroup>
     <PackageReference Include="GitHubActionsTestLogger" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" />
+    <PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="NServiceBus.Heartbeat" />
     <PackageReference Include="NServiceBus.SagaAudit" />
@@ -31,6 +35,9 @@
 
   <ItemGroup>
     <Compile Include="..\ServiceControl.AcceptanceTests\**\*.cs" LinkBase="Shared" />
+    <Compile Include="..\ServiceControl.Migration.AcceptanceTests\**\*.cs" LinkBase="Migration" />
+    <Compile Include="..\ServiceControl.Persistence.Tests\IngestedFailure.cs" />
+    <Compile Include="..\ServiceControl.Persistence.Tests\EFCore\InMemoryBodyStoragePersistence.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\PostgreSqlSharedContainer.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\TestSchema.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\StopSharedPostgreSql.cs" />

src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj

@@ -12,6 +12,9 @@
   <ItemGroup>
     <ProjectReference Include="..\ServiceControl.AcceptanceTesting\ServiceControl.AcceptanceTesting.csproj" />
     <ProjectReference Include="..\ServiceControl.Persistence.EFCore.SqlServer\ServiceControl.Persistence.EFCore.SqlServer.csproj" />
+    <!-- Built so its manifest is where DevelopmentPersistenceLocations looks; not copied, or its persistence.manifest would collide with this project's own. -->
+    <ProjectReference Include="..\ServiceControl.Persistence.RavenDB\ServiceControl.Persistence.RavenDB.csproj" ReferenceOutputAssembly="false" Private="false" />
+    <ProjectReference Include="..\ServiceControl.RavenDB\ServiceControl.RavenDB.csproj" />
     <ProjectReference Include="..\ServiceControl.Transports.Learning\ServiceControl.Transports.Learning.csproj" />
     <ProjectReference Include="..\ServiceControl\ServiceControl.csproj" />
     <ProjectReference Include="..\TestHelper\TestHelper.csproj" />
@@ -20,6 +23,7 @@
   <ItemGroup>
     <PackageReference Include="GitHubActionsTestLogger" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" />
+    <PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="NServiceBus.Heartbeat" />
     <PackageReference Include="NServiceBus.SagaAudit" />
@@ -31,6 +35,9 @@
 
   <ItemGroup>
     <Compile Include="..\ServiceControl.AcceptanceTests\**\*.cs" LinkBase="Shared" />
+    <Compile Include="..\ServiceControl.Migration.AcceptanceTests\**\*.cs" LinkBase="Migration" />
+    <Compile Include="..\ServiceControl.Persistence.Tests\IngestedFailure.cs" />
+    <Compile Include="..\ServiceControl.Persistence.Tests\EFCore\InMemoryBodyStoragePersistence.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.SqlServer\SqlServerSharedContainer.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.SqlServer\TestSchema.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.SqlServer\StopSharedSqlServer.cs" />

src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs

@@ -29,10 +29,9 @@ namespace ServiceControl.AcceptanceTests.Recoverability
     using ServiceControl.Infrastructure;
     using ServiceControl.MessageFailures;
     using ServiceControl.Operations;
+    using ServiceControl.Persistence.DataMigration;
     using ServiceControl.Persistence.EFCore.DbContexts;
     using ServiceControl.Persistence.EFCore.Entities;
-    using ServiceControl.Persistence.EFCore.Infrastructure;
-    using ServiceControl.Recoverability;
     using ServiceControl.Transports;
 
     class When_hosting_error_ingestion_only : AcceptanceTest
@@ -97,9 +96,36 @@ public async Task Should_default_the_concurrency_to_10_when_none_is_configured()
             "InternalCustomChecksHostedService",    // reports this node's ingestion health to the database
             "MetricsReporterHostedService",
             "HealthCheckPublisherHostedService",  // inert, no IHealthCheckPublisher is registered
-            "ExternalIntegrationRequestsDataStore"  // its drain is inert here, nothing calls Subscribe
+            "ExternalIntegrationRequestsDataStore",  // its drain is inert here, nothing calls Subscribe
+            "FinishedCopyBeforeAnIngestionNodeOpens"  // keeps this node out of a database a copy has not finished filling
         ];
 
+        [Test]
+        public async Task Should_refuse_to_start_while_a_copy_into_the_database_is_unfinished()
+        {
+            var settings = await CreateSettings();
+
+            await new SetupCommand().Execute(new HostArguments([]), settings);
+
+            var host = ErrorIngestionOnlyCommand.BuildHost(settings);
+
+            try
+            {
+                await host.Services.GetRequiredService<IMigrationCheckpointStore>().Upsert(
+                    new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Halted, null, 0, 0, null, null, null, null, null, "The source stopped responding."));
+
+                var exception = Assert.ThrowsAsync<Exception>(() => host.StartAsync());
+
+                Assert.That(exception.Message, Does.Contain("has not finished")
+                    .And.Contain(MigrationCategoryIds.KnownEndpoints)
+                    .And.Contain("--migration-abandon KnownEndpoints"));
+            }
+            finally
+            {
+                await host.DisposeAsync();
+            }
+        }
+
         [Test]
         public void Should_refuse_to_start_against_unsupported_storage()
         {
@@ -204,6 +230,7 @@ public async Task Should_serve_health_over_https_with_the_configured_certificate
                 await new SetupCommand().Execute(new HostArguments([]), settings);
 
                 host = ErrorIngestionOnlyCommand.BuildHost(settings);
+                host.Urls.Clear();
                 host.Urls.Add("https://127.0.0.1:0");
 
                 await host.StartAsync();

src/ServiceControl.Migration.AcceptanceTests/.editorconfig

@@ -0,0 +1,6 @@
+[*.cs]
+
+# Justification: Test project
+dotnet_diagnostic.CA2007.severity = none
+dotnet_diagnostic.PS0013.severity = none
+dotnet_diagnostic.PS0018.severity = none