feat: import leftover student fields into notes and show them on the roster

Store leftover CSV columns on hidden student notes, move catalog import to /events/import, and persist Students index columns from Chapter Settings.

Co-authored-by: Cursor <cursoragent@cursor.com>

WebApp/Services/ChapterSettingsWriter.cs

@@ -1,4 +1,5 @@
 using System.Text.Json;
+using System.Text.Json.Nodes;
 using WebApp.Models;
 
 namespace WebApp.Services;
@@ -8,6 +9,11 @@ namespace WebApp.Services;
 /// </summary>
 public class ChapterSettingsWriter : IChapterSettingsWriter
 {
+    private static readonly JsonSerializerOptions JsonOptions = new()
+    {
+        WriteIndented = true
+    };
+
     private readonly IWebHostEnvironment _environment;
     private readonly IConfiguration _configuration;
     private readonly ILogger<ChapterSettingsWriter> _logger;
@@ -29,35 +35,41 @@ public class ChapterSettingsWriter : IChapterSettingsWriter
         var appSettingsPath = GetAppSettingsPath();
         var dataDir = Path.GetDirectoryName(appSettingsPath);
         if (dataDir != null && !Directory.Exists(dataDir))
-        {
             Directory.CreateDirectory(dataDir);
-        }
 
-        Dictionary<string, object?> root;
+        JsonObject root;
         if (File.Exists(appSettingsPath))
         {
             var existingJson = await File.ReadAllTextAsync(appSettingsPath, cancellationToken);
-            root = JsonSerializer.Deserialize<Dictionary<string, object?>>(existingJson)
-                   ?? [];
+            root = JsonNode.Parse(existingJson) as JsonObject ?? [];
         }
         else
         {
             root = [];
         }
 
-        root["ChapterSettings"] = settings;
+        var incoming = JsonSerializer.SerializeToNode(settings, JsonOptions) as JsonObject ?? [];
+        if (root["ChapterSettings"] is JsonObject existing)
+        {
+            foreach (var property in incoming)
+                existing[property.Key] = property.Value?.DeepClone();
+        }
+        else
+        {
+            root["ChapterSettings"] = incoming;
+        }
 
-        var options = new JsonSerializerOptions { WriteIndented = true };
-        var json = JsonSerializer.Serialize(root, options);
-        await File.WriteAllTextAsync(appSettingsPath, json, cancellationToken);
+        await File.WriteAllTextAsync(appSettingsPath, root.ToJsonString(JsonOptions), cancellationToken);
+
+        if (_configuration is IConfigurationRoot configurationRoot)
+            configurationRoot.Reload();
 
         _logger.LogInformation("Chapter settings written to {Path}", appSettingsPath);
     }
 
     public async Task UpdateCompetitionYearAsync(string competitionYear, CancellationToken cancellationToken = default)
     {
-        var settings = _configuration.GetSection("ChapterSettings").Get<ChapterSettings>()
-                       ?? new ChapterSettings();
+        var settings = ChapterSettings.FromConfiguration(_configuration);
         settings.CompetitionYear = competitionYear;
         await WriteAsync(settings, cancellationToken);
     }

WebApp/Services/INotesService.cs

@@ -22,6 +22,26 @@ public interface INotesService
     /// <returns>The note with title "#{pageIdentifier}" or null if not found</returns>
     Task<Note?> GetPageNoteAsync(string pageIdentifier);
 
+    /// <summary>
+    /// Gets the system note for a student, or null if none exists.
+    /// </summary>
+    Task<Note?> GetStudentNoteAsync(int studentId);
+
+    /// <summary>
+    /// Gets system notes for the given student ids, keyed by student id.
+    /// </summary>
+    Task<IReadOnlyDictionary<int, Note>> GetStudentNotesAsync(IEnumerable<int> studentIds);
+
+    /// <summary>
+    /// Soft-deletes system notes for the given student ids.
+    /// </summary>
+    Task SoftDeleteStudentNotesAsync(IEnumerable<int> studentIds, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Distinct Field names from student note <c>## Imported fields</c> tables.
+    /// </summary>
+    Task<IReadOnlyList<string>> GetImportedFieldNamesAsync(CancellationToken cancellationToken = default);
+
     /// <summary>
     /// Gets all history entries for a note.
     /// </summary>

WebApp/Services/IStudentNotesImportSaveService.cs

@@ -0,0 +1,22 @@
+using Core.Models;
+
+namespace WebApp.Services;
+
+/// <summary>
+/// Persists merged student field notes.
+/// </summary>
+public interface IStudentNotesImportSaveService
+{
+    Task<StudentNotesImportSaveResult> SaveAsync(
+        StudentNotesImportResult parseResult,
+        CancellationToken cancellationToken = default);
+}
+
+public class StudentNotesImportSaveResult
+{
+    public int NotesCreated { get; set; }
+
+    public int NotesUpdated { get; set; }
+
+    public int StudentsUnchanged { get; set; }
+}

WebApp/Services/NotesService.cs

@@ -1,4 +1,5 @@
 using Core.Entities;
+using Core.Notes;
 using Core.Services;
 using Data;
 using Microsoft.EntityFrameworkCore;
@@ -46,8 +47,9 @@ public class NotesService : INotesService
         }
 
         return await query
-            .OrderBy(n => n.Title != null && n.Title.StartsWith("#") ? 1 : 0) // Non-page notes first (0), page notes last (1)
-            .ThenByDescending(n => n.UpdatedAt) // Within each group, order by most recently updated
+            .Where(n => n.Title == null || !n.Title.StartsWith("#Student:"))
+            .OrderBy(n => n.Title != null && n.Title.StartsWith("#") ? 1 : 0)
+            .ThenByDescending(n => n.UpdatedAt)
             .ToListAsync();
     }
 
@@ -67,6 +69,84 @@ public class NotesService : INotesService
             .FirstOrDefaultAsync();
     }
 
+    public async Task<Note?> GetStudentNoteAsync(int studentId)
+    {
+        var title = _noteNamingService.GetStudentNoteTitle(studentId);
+        return await _context.Notes
+            .AsNoTracking()
+            .Where(n => n.Title == title && !n.IsDeleted)
+            .FirstOrDefaultAsync();
+    }
+
+    public async Task<IReadOnlyDictionary<int, Note>> GetStudentNotesAsync(IEnumerable<int> studentIds)
+    {
+        var ids = studentIds.Distinct().ToList();
+        if (ids.Count == 0)
+            return new Dictionary<int, Note>();
+
+        var titles = ids.Select(_noteNamingService.GetStudentNoteTitle).ToList();
+        var notes = await _context.Notes
+            .AsNoTracking()
+            .Where(n => titles.Contains(n.Title) && !n.IsDeleted)
+            .ToListAsync();
+
+        Dictionary<int, Note> byStudentId = [];
+        foreach (var note in notes)
+        {
+            if (_noteNamingService.TryParseStudentNoteId(note.Title, out var studentId))
+                byStudentId[studentId] = note;
+        }
+
+        return byStudentId;
+    }
+
+    public async Task SoftDeleteStudentNotesAsync(IEnumerable<int> studentIds, CancellationToken cancellationToken = default)
+    {
+        var ids = studentIds.Distinct().ToList();
+        if (ids.Count == 0)
+            return;
+
+        var titles = ids.Select(_noteNamingService.GetStudentNoteTitle).ToList();
+        var notes = await _context.Notes
+            .Where(n => titles.Contains(n.Title) && !n.IsDeleted)
+            .ToListAsync(cancellationToken);
+
+        if (notes.Count == 0)
+            return;
+
+        var userEmail = GetCurrentUserEmail();
+        var now = DateTime.UtcNow;
+        foreach (var note in notes)
+        {
+            _context.NoteHistories.Add(new NoteHistory
+            {
+                NoteId = note.Id,
+                Title = note.Title,
+                Content = note.Content,
+                ModifiedBy = userEmail,
+                ModifiedAt = now,
+                ChangeType = "Soft Deleted"
+            });
+            note.IsDeleted = true;
+            note.UpdatedAt = now;
+            note.LastModifiedBy = userEmail;
+        }
+
+        await _context.SaveChangesAsync(cancellationToken);
+        _logger.LogInformation("Soft-deleted {Count} student note(s)", notes.Count);
+    }
+
+    public async Task<IReadOnlyList<string>> GetImportedFieldNamesAsync(CancellationToken cancellationToken = default)
+    {
+        var contents = await _context.Notes
+            .AsNoTracking()
+            .Where(n => !n.IsDeleted && n.Title != null && n.Title.StartsWith("#Student:"))
+            .Select(n => n.Content)
+            .ToListAsync(cancellationToken);
+
+        return ImportedFieldsTable.DistinctFieldNames(contents);
+    }
+
     public async Task<IEnumerable<NoteHistory>> GetNoteHistoryAsync(int noteId)
     {
         return await _context.NoteHistories
@@ -253,9 +333,9 @@ public class NotesService : INotesService
     {
         return await _context.Notes
             .AsNoTracking()
-            .Where(n => n.IsDeleted)
-            .OrderBy(n => n.Title != null && n.Title.StartsWith("#") ? 1 : 0) // Non-page notes first (0), page notes last (1)
-            .ThenByDescending(n => n.UpdatedAt) // Within each group, order by most recently updated
+            .Where(n => n.IsDeleted && (n.Title == null || !n.Title.StartsWith("#Student:")))
+            .OrderBy(n => n.Title != null && n.Title.StartsWith("#") ? 1 : 0)
+            .ThenByDescending(n => n.UpdatedAt)
             .ToListAsync();
     }
 

WebApp/Services/StudentNotesImportSaveService.cs

@@ -0,0 +1,64 @@
+using Core.Models;
+using Core.Services;
+
+namespace WebApp.Services;
+
+/// <summary>
+/// Creates or updates #Student:{id} notes when imported fields actually change.
+/// </summary>
+public class StudentNotesImportSaveService : IStudentNotesImportSaveService
+{
+    private readonly INotesService _notesService;
+    private readonly INoteNamingService _noteNamingService;
+    private readonly ILogger<StudentNotesImportSaveService> _logger;
+
+    public StudentNotesImportSaveService(
+        INotesService notesService,
+        INoteNamingService noteNamingService,
+        ILogger<StudentNotesImportSaveService> logger)
+    {
+        _notesService = notesService;
+        _noteNamingService = noteNamingService;
+        _logger = logger;
+    }
+
+    /// <inheritdoc />
+    public async Task<StudentNotesImportSaveResult> SaveAsync(
+        StudentNotesImportResult parseResult,
+        CancellationToken cancellationToken = default)
+    {
+        var result = new StudentNotesImportSaveResult();
+        var existingNotes = await _notesService.GetStudentNotesAsync(
+            parseResult.Matches.Select(m => m.Student.Id));
+        var actions = StudentNotesImportPlan.Create(parseResult, existingNotes.Keys.ToHashSet());
+        result.StudentsUnchanged = parseResult.Matches.Count - actions.Count;
+
+        foreach (var action in actions)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            if (action.Kind == StudentNotePersistKind.Update
+                && existingNotes.TryGetValue(action.StudentId, out var existing))
+            {
+                existing.Content = action.Markdown;
+                await _notesService.UpdateNoteAsync(existing);
+                result.NotesUpdated++;
+                continue;
+            }
+
+            await _notesService.CreateNoteAsync(new Core.Entities.Note
+            {
+                Title = _noteNamingService.GetStudentNoteTitle(action.StudentId),
+                Content = action.Markdown
+            });
+            result.NotesCreated++;
+        }
+
+        _logger.LogInformation(
+            "Student notes import saved. Created={Created}, Updated={Updated}, Unchanged={Unchanged}",
+            result.NotesCreated,
+            result.NotesUpdated,
+            result.StudentsUnchanged);
+
+        return result;
+    }
+}

WebApp/Services/YearRolloverService.cs

@@ -12,17 +12,20 @@ public class YearRolloverService : IYearRolloverService
     private readonly AppDbContext _context;
     private readonly IDatabaseBackupService _backupService;
     private readonly IChapterSettingsWriter _chapterSettingsWriter;
+    private readonly INotesService _notesService;
     private readonly ILogger<YearRolloverService> _logger;
 
     public YearRolloverService(
         AppDbContext context,
         IDatabaseBackupService backupService,
         IChapterSettingsWriter chapterSettingsWriter,
+        INotesService notesService,
         ILogger<YearRolloverService> logger)
     {
         _context = context;
         _backupService = backupService;
         _chapterSettingsWriter = chapterSettingsWriter;
+        _notesService = notesService;
         _logger = logger;
     }
 
@@ -107,6 +110,8 @@ public class YearRolloverService : IYearRolloverService
                         $"Roster changed since the wizard loaded ({unexpected.Count} unexpected student(s): {names}). Reload the page and try again.");
                 }
 
+                await _notesService.SoftDeleteStudentNotesAsync(removalIds, cancellationToken);
+
                 _context.Students.RemoveRange(toRemove);
 
                 var promotionById = plan.Promotions.ToDictionary(p => p.Student.Id);