From a2e17e2e40625400699cd5a5b32284ed4f5aa84d Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Tue, 29 Sep 2026 20:13:32 +0700 Subject: [PATCH 1/3] fix(plugins): detect JSON import fields from every row and read JSON Lines by byte --- CHANGELOG.md | 4 + .../JSONImportPlugin/JSONFieldSurvey.swift | 102 +++++++++++++ .../JSONImportPlugin/JSONImportParsing.swift | 126 ++++++---------- .../JSONImportPlugin/JSONImportPlugin.swift | 21 ++- Plugins/JSONImportPlugin/JSONLineReader.swift | 88 +++++++++++ .../Core/Plugins/ImportFieldDetection.swift | 27 ++++ TablePro/Views/Import/RowImportSheet.swift | 105 +++++++------ .../Plugins/ImportFieldDetectionTests.swift | 74 +++++++++ .../JSONImportFieldDetectionTests.swift | 142 ++++++++++++++++++ .../Plugins/JSONImportPluginTests.swift | 59 ++++++-- .../Plugins/JSONImportSkipTests.swift | 138 +++++++++++++++-- .../Plugins/JSONLineReaderTests.swift | 106 +++++++++++++ docs/features/import-export.mdx | 2 + project.yml | 2 + 14 files changed, 836 insertions(+), 160 deletions(-) create mode 100644 Plugins/JSONImportPlugin/JSONFieldSurvey.swift create mode 100644 Plugins/JSONImportPlugin/JSONLineReader.swift create mode 100644 TablePro/Core/Plugins/ImportFieldDetection.swift create mode 100644 TableProTests/Core/Plugins/ImportFieldDetectionTests.swift create mode 100644 TableProTests/Plugins/JSONImportFieldDetectionTests.swift create mode 100644 TableProTests/Plugins/JSONLineReaderTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 40eb3555b3..05bc2c73c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - iOS row editor saving the placeholder of a long text or binary value over the full value. - Explain Analyze running write statements on Read-Only connections and skipping the Alert and Safe Mode confirmation. - **Local only** connections taking edits and deletions made on another device. +- JSON import leaving out fields first seen after row 200, and typing columns from those rows alone. +- No fields found in a JSON Lines file with CRLF line endings or multi-byte text near its first 256 KB. +- JSON Lines rows with U+2028, U+2029 or U+0085 in a string failing to import, and invalid UTF-8 imported as U+FFFD. +- JSON Lines import stalling while GitHub Copilot is enabled. ## [0.76.1] - 2026-09-29 diff --git a/Plugins/JSONImportPlugin/JSONFieldSurvey.swift b/Plugins/JSONImportPlugin/JSONFieldSurvey.swift new file mode 100644 index 0000000000..636afd94f9 --- /dev/null +++ b/Plugins/JSONImportPlugin/JSONFieldSurvey.swift @@ -0,0 +1,102 @@ +// +// JSONFieldSurvey.swift +// JSONImportPlugin +// + +import Foundation +import TableProPluginKit + +/// Every field any row of the file names, each with the first value it holds and a type that fits +/// every value it holds. It keeps one small record per field rather than the values themselves, +/// so it can read every row of a file of any size. +struct JSONFieldSurvey { + private var profiles: [String: JSONFieldProfile] = [:] + + mutating func add(_ row: NSDictionary) { + row.enumerateKeysAndObjects { key, value, _ in + guard let name = key as? String else { return } + profiles[name, default: JSONFieldProfile()].add(value) + } + } + + var fields: [PluginImportField] { + profiles.keys.sorted().map { name in + let profile = profiles[name] ?? JSONFieldProfile() + return PluginImportField( + name: name, + sampleValue: profile.sampleValue, + inferredType: profile.kinds.inferredType + ) + } + } +} + +struct JSONFieldProfile { + private(set) var sampleValue: String? + private(set) var kinds = JSONValueKinds() + + mutating func add(_ value: Any) { + let kind = JSONValueKind(of: value) + guard kind != .null else { return } + if sampleValue == nil { + sampleValue = JSONImportParsing.sampleString(value) + } + kinds.add(kind) + } +} + +struct JSONValueKinds { + private var sawValue = false + private var allNested = true + private var allBoolean = true + private var allInteger = true + private var allNumber = true + + mutating func add(_ kind: JSONValueKind) { + sawValue = true + if kind != .nested { allNested = false } + if kind != .boolean { allBoolean = false } + if kind != .integer { allInteger = false } + if kind != .integer, kind != .real { allNumber = false } + } + + var inferredType: PluginImportFieldType { + guard sawValue else { return .text } + if allNested { return .json } + if allBoolean { return .boolean } + if allInteger { return .integer } + if allNumber { return .real } + return .text + } +} + +/// Reads a parsed JSON value's kind from its Core Foundation type. A survey classifies every value +/// in the file, and a Swift `is` or `as?` cast from `Any` costs about ten times as much. +enum JSONValueKind: Equatable { + case null + case nested + case boolean + case integer + case real + case other + + init(of value: Any) { + let object = value as AnyObject + switch CFGetTypeID(object) { + case CFNullGetTypeID(): + self = .null + case CFArrayGetTypeID(), CFDictionaryGetTypeID(): + self = .nested + case CFBooleanGetTypeID(): + self = .boolean + case CFNumberGetTypeID(): + guard let number = object as? NSNumber else { + self = .other + return + } + self = CFNumberIsFloatType(number) ? .real : .integer + default: + self = .other + } + } +} diff --git a/Plugins/JSONImportPlugin/JSONImportParsing.swift b/Plugins/JSONImportPlugin/JSONImportParsing.swift index 59a4d3c8e1..f0fb4ee9d9 100644 --- a/Plugins/JSONImportPlugin/JSONImportParsing.swift +++ b/Plugins/JSONImportPlugin/JSONImportParsing.swift @@ -16,23 +16,32 @@ enum JSONImportParsing { ["jsonl", "ndjson"].contains(url.pathExtension.lowercased()) } - static func parseRow(fromLine line: String) throws -> [String: PluginCellValue] { - let object = try JSONSerialization.jsonObject(with: Data(line.utf8)) - guard let dict = object as? [String: Any] else { + static func parseRow(fromLine line: Data) throws -> [String: PluginCellValue]? { + try object(fromLine: line).map(convertRow) + } + + static func object(fromLine line: Data) throws -> NSDictionary? { + guard !isBlank(line) else { return nil } + let object = try JSONSerialization.jsonObject(with: line) + guard let dict = object as? NSDictionary else { throw PluginImportError.importFailed("Each line must be a JSON object") } - return convertRow(dict) + return dict } - static func parseRows(at url: URL, targetTable: String?) throws -> [[String: Any]] { + private static func isBlank(_ line: Data) -> Bool { + line.allSatisfy { $0 == 0x20 || $0 == 0x09 || $0 == 0x0D } + } + + static func parseRows(at url: URL, targetTable: String?) throws -> [NSDictionary] { let data = try Data(contentsOf: url) let object = try JSONSerialization.jsonObject(with: data) return try extractRows(from: object, targetTable: targetTable) } - static func extractRows(from object: Any, targetTable: String?) throws -> [[String: Any]] { + static func extractRows(from object: Any, targetTable: String?) throws -> [NSDictionary] { if let array = object as? [Any] { - return array.compactMap { $0 as? [String: Any] } + return array.compactMap { $0 as? NSDictionary } } guard let dict = object as? [String: Any] else { @@ -41,19 +50,19 @@ enum JSONImportParsing { let tables = dict.compactMapValues { value -> [Any]? in guard let array = value as? [Any] else { return nil } - return array.allSatisfy { $0 is [String: Any] } ? array : nil + return array.allSatisfy { $0 is NSDictionary } ? array : nil } let isTableWrapper = !tables.isEmpty && tables.count == dict.count guard isTableWrapper else { - return [dict] + return [dict as NSDictionary] } if let targetTable, let match = matchTable(in: tables, to: targetTable) { - return match.compactMap { $0 as? [String: Any] } + return match.compactMap { $0 as? NSDictionary } } if tables.count == 1, let only = tables.values.first { - return only.compactMap { $0 as? [String: Any] } + return only.compactMap { $0 as? NSDictionary } } throw PluginImportError.importFailed("The file contains multiple tables and none matches the target table") } @@ -68,8 +77,14 @@ enum JSONImportParsing { return suffix?.value } - static func convertRow(_ row: [String: Any]) -> [String: PluginCellValue] { - row.mapValues(cellValue(from:)) + static func convertRow(_ row: NSDictionary) -> [String: PluginCellValue] { + var converted: [String: PluginCellValue] = [:] + converted.reserveCapacity(row.count) + row.enumerateKeysAndObjects { key, value, _ in + guard let name = key as? String else { return } + converted[name] = cellValue(from: value) + } + return converted } static func cellValue(from json: Any) -> PluginCellValue { @@ -94,81 +109,32 @@ enum JSONImportParsing { // MARK: - Source introspection - static func sampleRawRows(at url: URL, targetTable: String?, limit: Int) throws -> [[String: Any]] { - if isLineDelimited(url) { - let handle = try FileHandle(forReadingFrom: url) - defer { try? handle.close() } - let text = String(bytes: handle.readData(ofLength: 256 * 1_024), encoding: .utf8) ?? "" - var rows: [[String: Any]] = [] - for line in text.split(separator: "\n") where rows.count < limit { - let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { continue } - if let object = try? JSONSerialization.jsonObject(with: Data(trimmed.utf8)) as? [String: Any] { - rows.append(object) - } - } - return rows + static func detectFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { + guard isLineDelimited(url) else { + return try detectFields(in: try parseRows(at: url, targetTable: targetTable)) } - let object = try JSONSerialization.jsonObject(with: Data(contentsOf: url)) - return Array(try extractRows(from: object, targetTable: targetTable).prefix(limit)) - } - - static func detectFields(in rows: [[String: Any]]) -> [PluginImportField] { - var names: [String] = [] - var seen = Set() - var valuesByField: [String: [Any]] = [:] - for row in rows { - for (key, value) in row { - if seen.insert(key).inserted { names.append(key) } - valuesByField[key, default: []].append(value) + var survey = JSONFieldSurvey() + var lines = try JSONLineReader(url: url) + defer { lines.close() } + while let line = try lines.next() { + autoreleasepool { + guard let row = try? object(fromLine: line) else { return } + survey.add(row) } } - return names.sorted().map { name in - let nonNull = (valuesByField[name] ?? []).filter { !($0 is NSNull) } - return PluginImportField( - name: name, - sampleValue: nonNull.first.map(sampleString), - inferredType: inferType(from: nonNull) - ) - } + return survey.fields } - static func inferType(from values: [Any]) -> PluginImportFieldType { - guard !values.isEmpty else { return .text } - var allNested = true - var allBoolean = true - var allInteger = true - var allNumber = true - for value in values { - if value is [Any] || value is [String: Any] { - allBoolean = false - allInteger = false - allNumber = false - } else { - allNested = false - if let number = value as? NSNumber { - if CFGetTypeID(number) == CFBooleanGetTypeID() { - allInteger = false - allNumber = false - } else { - allBoolean = false - if CFNumberIsFloatType(number) { allInteger = false } - } - } else { - allBoolean = false - allInteger = false - allNumber = false - } - } + static func detectFields(in rows: [NSDictionary]) throws -> [PluginImportField] { + var survey = JSONFieldSurvey() + for row in rows { + try Task.checkCancellation() + survey.add(row) } - if allNested { return .json } - if allBoolean { return .boolean } - if allInteger { return .integer } - if allNumber { return .real } - return .text + return survey.fields } - private static func sampleString(_ value: Any) -> String { + static func sampleString(_ value: Any) -> String { switch cellValue(from: value) { case .text(let string): return String(string.prefix(80)) case .bytes, .null: return "" diff --git a/Plugins/JSONImportPlugin/JSONImportPlugin.swift b/Plugins/JSONImportPlugin/JSONImportPlugin.swift index 19161b0b4a..5d4f5323a3 100644 --- a/Plugins/JSONImportPlugin/JSONImportPlugin.swift +++ b/Plugins/JSONImportPlugin/JSONImportPlugin.swift @@ -60,20 +60,20 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug var unreadableLineCount = 0 if JSONImportParsing.isLineDelimited(url) { progress.setEstimatedTotal(max(1, Int(source.fileSizeBytes() / 256))) - var lines = url.lines.makeAsyncIterator() - var lineNumber = 0 + var lines = try JSONLineReader(url: url, checkCancellation: progress.checkCancellation) + defer { lines.close() } let skipsErrors = settings.errorHandling == .skipAndContinue outcome = try await RowImportRunner.run( configuration: configuration, sink: sink, progress: progress ) { var batch: [RowImportRunner.Entry] = [] - while batch.count < Self.batchSize, let line = try await lines.next() { - lineNumber += 1 - let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { continue } + var linesRead = 0 + while linesRead < Self.batchSize, let line = try lines.next() { + linesRead += 1 + let lineNumber = lines.lineNumber do { - let row = try JSONImportParsing.parseRow(fromLine: trimmed) - guard !row.isEmpty else { continue } + let row = try autoreleasepool { try JSONImportParsing.parseRow(fromLine: line) } + guard let row, !row.isEmpty else { continue } batch.append((lineNumber, row)) } catch { guard skipsErrors else { throw error } @@ -87,7 +87,7 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug } } } - return batch.isEmpty ? nil : batch + return linesRead == 0 ? nil : batch } } else { let rawRows = try JSONImportParsing.parseRows(at: url, targetTable: sink.targetTable) @@ -119,7 +119,6 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug // MARK: - Source introspection func detectSourceFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { - let rows = try JSONImportParsing.sampleRawRows(at: url, targetTable: targetTable, limit: 200) - return JSONImportParsing.detectFields(in: rows) + try JSONImportParsing.detectFields(at: url, targetTable: targetTable) } } diff --git a/Plugins/JSONImportPlugin/JSONLineReader.swift b/Plugins/JSONImportPlugin/JSONLineReader.swift new file mode 100644 index 0000000000..4195b4f3be --- /dev/null +++ b/Plugins/JSONImportPlugin/JSONLineReader.swift @@ -0,0 +1,88 @@ +// +// JSONLineReader.swift +// JSONImportPlugin +// + +import Foundation + +/// Reads a JSON Lines file one line at a time, holding one chunk and the line in progress. +/// +/// A line ends at a 0x0A byte and nowhere else, and its bytes go to the JSON parser undecoded. +/// Decoding a fixed-size prefix as text failed whenever the prefix ended inside a multi-byte +/// character. `URL.lines` also ends a line at U+2028, U+2029 and U+0085, which JSON allows +/// unescaped inside a string, and it reads through `FileHandle.AsyncBytes`, whose one +/// process-wide queue a reader waiting on a quiet pipe elsewhere in the app holds. +/// +/// A stop is checked before every chunk rather than between lines, because one line can run to +/// the end of a file that holds no newline at all. +struct JSONLineReader { + static let defaultChunkSize = 1 << 20 + + private let handle: FileHandle + private let chunkSize: Int + private let checkCancellation: () throws -> Void + private var buffer = Data() + private var lineStart = 0 + private var searchedUpTo = 0 + private var reachedEnd = false + + private(set) var lineNumber = 0 + + init( + url: URL, + chunkSize: Int = Self.defaultChunkSize, + checkCancellation: @escaping () throws -> Void = { try Task.checkCancellation() } + ) throws { + handle = try FileHandle(forReadingFrom: url) + self.chunkSize = max(1, chunkSize) + self.checkCancellation = checkCancellation + } + + mutating func next() throws -> Data? { + while true { + if let newline = firstNewline() { + let line = buffer[lineStart.. Int? { + buffer.withUnsafeBytes { raw -> Int? in + guard searchedUpTo < raw.count, let base = raw.baseAddress else { return nil } + guard let found = memchr(base + searchedUpTo, 0x0A, raw.count - searchedUpTo) else { return nil } + return base.distance(to: UnsafeRawPointer(found)) + } + } + + private mutating func refill() throws { + try checkCancellation() + if lineStart > 0 { + buffer.removeSubrange(0.. [PluginImportField] { + let detection = Task.detached { + try plugin.detectSourceFields(at: url, targetTable: targetTable) + } + return try await withTaskCancellationHandler { + try await detection.value + } onCancel: { + detection.cancel() + } + } +} diff --git a/TablePro/Views/Import/RowImportSheet.swift b/TablePro/Views/Import/RowImportSheet.swift index 37c8f746b8..674c68257d 100644 --- a/TablePro/Views/Import/RowImportSheet.swift +++ b/TablePro/Views/Import/RowImportSheet.swift @@ -28,6 +28,16 @@ struct RowImportSheet: View { case newTable } + /// What the field list on screen is read for. Detection reads the whole file, so it runs only + /// for the destination on screen, and a request that changes or a sheet that closes cancels the + /// read still running for the old one. + private struct FieldDetectionRequest: Hashable { + let destination: Destination + let targetTable: String? + let detectionSignature: String + let attempt: Int + } + private struct FieldMapping: Identifiable { let field: PluginImportField var include: Bool @@ -74,8 +84,12 @@ struct RowImportSheet: View { @State private var proposedTableName: String = "" @State private var newColumns: [NewColumn] = [] @State private var newColumnsLoaded = false + /// The table `mappings` were built for. Switching destination and back keeps the user's edits + /// rather than reading the file again and replacing them with fresh matches. + @State private var mappedTable: String? @State private var isLoadingContext = false @State private var loadError: String? + @State private var detectionAttempt = 0 /// Moving focus here also selects the whole proposed name, measured rather than assumed: /// SwiftUI hands the field editor a full selection when `@FocusState` lands on text already in @@ -166,21 +180,23 @@ struct RowImportSheet: View { plugins: [currentPlugin as? any SettablePluginDiscoverable].compactMap { $0 }) suggestNewTableName() await loadTables() - await loadNewColumns() + } + .task(id: fieldDetectionRequest) { + await loadFields(for: fieldDetectionRequest) } .onChange(of: destination) { newValue in guard newValue == .newTable else { return } suggestNewTableName() newTableNameFocused = true } - .onChange(of: selectedTargetTable) { newValue in + .onChange(of: selectedTargetTable) { _ in mappings = [] targetColumns = [] - guard destination == .existingTable, let table = newValue else { return } - Task { await loadExistingContext(table: table) } + mappedTable = nil } .onChange(of: currentPlugin?.fieldDetectionSignature) { _ in - Task { await redetectFields() } + newColumnsLoaded = false + mappedTable = nil } .onDisappear { importTask?.cancel() @@ -369,24 +385,18 @@ struct RowImportSheet: View { Text(reason) } actions: { Button(String(localized: "Try Again")) { - Task { await retryLoad() } + retryLoad() } } } - @MainActor - private func retryLoad() async { + private func retryLoad() { loadError = nil newColumnsLoaded = false newColumns = [] mappings = [] - switch destination { - case .newTable: - await loadNewColumns() - case .existingTable: - guard let table = selectedTargetTable else { return } - await loadExistingContext(table: table) - } + mappedTable = nil + detectionAttempt += 1 } private func placeholder(_ message: String) -> some View { @@ -716,27 +726,39 @@ struct RowImportSheet: View { newTableName = suggestion } - /// `detectSourceFields` is synchronous and reads the file: the XLSX plugin materialises the - /// whole workbook, the CSV one reads a megabyte. Every state write stays on the main actor, - /// only the parse leaves it. - nonisolated private static func detectFields( - plugin: any ImportFormatPlugin, - at url: URL, - targetTable: String? - ) async throws -> [PluginImportField] { - try await Task.detached { - try plugin.detectSourceFields(at: url, targetTable: targetTable) - }.value + private var fieldDetectionRequest: FieldDetectionRequest { + FieldDetectionRequest( + destination: destination, + targetTable: destination == .existingTable ? selectedTargetTable : nil, + detectionSignature: currentPlugin?.fieldDetectionSignature ?? "", + attempt: detectionAttempt + ) } + /// A cancelled load leaves every piece of state to the load that replaced it, or to nobody once + /// the sheet has closed. Only the current request clears the loading indicator. @MainActor - private func loadNewColumns() async { - guard !newColumnsLoaded, let plugin = currentPlugin else { return } + private func loadFields(for request: FieldDetectionRequest) async { isLoadingContext = true loadError = nil - defer { isLoadingContext = false } + switch request.destination { + case .newTable: + await loadNewColumns() + case .existingTable: + if let table = request.targetTable { + await loadExistingContext(table: table) + } + } + guard !Task.isCancelled else { return } + isLoadingContext = false + } + + @MainActor + private func loadNewColumns() async { + guard !newColumnsLoaded, let plugin = currentPlugin else { return } do { - let fields = try await Self.detectFields(plugin: plugin, at: fileURL, targetTable: nil) + let fields = try await ImportFieldDetection.detectFields(plugin: plugin, at: fileURL, targetTable: nil) + guard !Task.isCancelled else { return } let serverVersion = DatabaseManager.shared.driver(for: connection.id)?.serverVersion newColumns = fields.map { field in NewColumn( @@ -755,6 +777,7 @@ struct RowImportSheet: View { } newColumnsLoaded = true } catch { + guard !Task.isCancelled else { return } loadError = error.localizedDescription Self.logger.warning("Failed to read import fields: \(error.publicLogShape, privacy: .public)") } @@ -762,41 +785,29 @@ struct RowImportSheet: View { @MainActor private func loadExistingContext(table: String) async { - guard let plugin = currentPlugin, + guard mappedTable != table, let plugin = currentPlugin, DatabaseManager.shared.browseScope(for: connection.id) != nil else { return } - isLoadingContext = true - loadError = nil - defer { isLoadingContext = false } do { let columns = try await DatabaseManager.shared.withBrowseMetadataDriver( connectionId: connection.id ) { driver in try await driver.fetchColumns(table: table) }.map(\.name) - let fields = try await Self.detectFields(plugin: plugin, at: fileURL, targetTable: table) + let fields = try await ImportFieldDetection.detectFields(plugin: plugin, at: fileURL, targetTable: table) + guard !Task.isCancelled else { return } targetColumns = columns mappings = fields.map { field in let match = columns.first { $0.caseInsensitiveCompare(field.name) == .orderedSame } return FieldMapping(field: field, include: match != nil, targetColumn: match) } + mappedTable = table } catch { + guard !Task.isCancelled else { return } loadError = error.localizedDescription Self.logger.warning("Failed to read import fields: \(error.publicLogShape, privacy: .public)") } } - @MainActor - private func redetectFields() async { - switch destination { - case .existingTable: - guard let table = selectedTargetTable else { return } - await loadExistingContext(table: table) - case .newTable: - newColumnsLoaded = false - await loadNewColumns() - } - } - // MARK: - Import private func performImport() { diff --git a/TableProTests/Core/Plugins/ImportFieldDetectionTests.swift b/TableProTests/Core/Plugins/ImportFieldDetectionTests.swift new file mode 100644 index 0000000000..202cd509c9 --- /dev/null +++ b/TableProTests/Core/Plugins/ImportFieldDetectionTests.swift @@ -0,0 +1,74 @@ +// +// ImportFieldDetectionTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class BlockingDetectionPlugin: ImportFormatPlugin, @unchecked Sendable { + static let pluginName = "Blocking Detection" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Detects fields until it is cancelled" + static let formatId = "blocking-detection" + static let formatDisplayName = "Blocking" + static let acceptedFileExtensions = ["blocking"] + static let iconName = "doc" + + private let lock = NSLock() + private var started = false + private var cancelled = false + + required init() {} + + var hasStarted: Bool { lock.withLock { started } } + var sawCancellation: Bool { lock.withLock { cancelled } } + + func performImport( + source: any PluginImportSource, + sink: any PluginImportDataSink, + progress: PluginImportProgress + ) async throws -> PluginImportResult { + PluginImportResult(executedStatements: 0, executionTime: 0) + } + + func detectSourceFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { + lock.withLock { started = true } + let deadline = Date().addingTimeInterval(10) + while !Task.isCancelled, Date() < deadline { + usleep(1_000) + } + guard Task.isCancelled else { return [] } + lock.withLock { cancelled = true } + throw CancellationError() + } +} + +struct ImportFieldDetectionTests { + /// The import sheet reads the file from a detached task. Nothing cancelled that task, so closing + /// the sheet left a whole-file read running to the end. + @Test("Cancelling the caller cancels a detection already reading the file") + func cancellingTheCallerStopsTheRead() async throws { + let plugin = BlockingDetectionPlugin() + let caller = Task { + try await ImportFieldDetection.detectFields( + plugin: plugin, + at: URL(fileURLWithPath: "/dev/null"), + targetTable: nil + ) + } + for _ in 0..<500 where !plugin.hasStarted { + try await Task.sleep(nanoseconds: 10_000_000) + } + try #require(plugin.hasStarted) + + caller.cancel() + + await #expect(throws: CancellationError.self) { + _ = try await caller.value + } + #expect(plugin.sawCancellation) + } +} diff --git a/TableProTests/Plugins/JSONImportFieldDetectionTests.swift b/TableProTests/Plugins/JSONImportFieldDetectionTests.swift new file mode 100644 index 0000000000..acaf9e23dc --- /dev/null +++ b/TableProTests/Plugins/JSONImportFieldDetectionTests.swift @@ -0,0 +1,142 @@ +// +// JSONImportFieldDetectionTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct JSONImportFieldDetectionTests { + private func write(_ bytes: Data, fileExtension: String) throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("json-import-detection-\(UUID().uuidString).\(fileExtension)") + try bytes.write(to: url) + return url + } + + private func detect(_ bytes: Data, fileExtension: String, targetTable: String? = nil) throws -> [PluginImportField] { + let url = try write(bytes, fileExtension: fileExtension) + defer { try? FileManager.default.removeItem(at: url) } + return try JSONImportParsing.detectFields(at: url, targetTable: targetTable) + } + + private func detectLines(_ lines: [String]) throws -> [PluginImportField] { + try detect(Data(lines.map { $0 + "\n" }.joined().utf8), fileExtension: "ndjson") + } + + private func field(_ name: String, in fields: [PluginImportField]) throws -> PluginImportField { + try #require(fields.first { $0.name == name }) + } + + /// TablePro's own JSON export leaves a null key out of a row, so a column can first appear + /// deep into the file. Detection used to read 200 rows and never offer it. + @Test("A key first seen after row 200 of a JSON Lines file is detected") + func lateKeyInJSONLines() throws { + var lines = (1...200).map { #"{"id":\#($0)}"# } + lines.append(#"{"id":201,"note":"late","score":1.5}"#) + let fields = try detectLines(lines) + #expect(fields.map(\.name) == ["id", "note", "score"]) + #expect(try field("note", in: fields).sampleValue == "late") + #expect(try field("note", in: fields).inferredType == .text) + #expect(try field("score", in: fields).inferredType == .real) + } + + @Test("A key first seen after element 200 of a JSON array is detected") + func lateKeyInJSONArray() throws { + var elements = (1...250).map { #"{"id":\#($0)}"# } + elements[229] = #"{"id":230,"deleted_at":"2026-01-02"}"# + let fields = try detect(Data("[\(elements.joined(separator: ","))]".utf8), fileExtension: "json") + #expect(fields.map(\.name) == ["deleted_at", "id"]) + #expect(try field("deleted_at", in: fields).sampleValue == "2026-01-02") + } + + @Test("A table-keyed file reports the fields of the target table") + func tableKeyedFileUsesTheTargetTable() throws { + let json = #"{"users":[{"id":1}],"orders":[{"id":1},{"id":2,"total":2.5}]}"# + let fields = try detect(Data(json.utf8), fileExtension: "json", targetTable: "orders") + #expect(fields.map(\.name) == ["id", "total"]) + } + + @Test("A field's type fits every value in the file, not only the first rows") + func typeFitsEveryValue() throws { + var lines = (1...300).map { #"{"code":\#($0)}"# } + lines.append(#"{"code":"A-301"}"#) + #expect(try field("code", in: detectLines(lines)).inferredType == .text) + } + + @Test("A key first seen past the first megabyte is detected") + func lateKeyPastTheFirstChunk() throws { + let padding = String(repeating: "x", count: 1_000) + var lines = Array(repeating: #"{"pad":"\#(padding)"}"#, count: 1_200) + lines.append(#"{"pad":"y","city":"Hà Nội"}"#) + let fields = try detectLines(lines) + #expect(try field("city", in: fields).sampleValue == "Hà Nội") + } + + /// Detection decoded the first 262,144 bytes as UTF-8 text, which fails outright when that + /// byte falls inside a character, so the whole file read as having no fields. + @Test("Text whose characters straddle the old prefix and the read chunk is detected") + func multiByteCharacterAtTheOldPrefixEnd() throws { + var bytes = Data() + let namePrefixLength = #"{"name":""#.utf8.count + func appendLine(_ line: String) { + bytes.append(Data((line + "\n").utf8)) + } + func appendPadding(untilNextLineStartsAt start: Int) { + let fixedLength = #"{"pad":""}"#.utf8.count + 1 + appendLine(#"{"pad":""# + String(repeating: "x", count: start - bytes.count - fixedLength) + #""}"#) + } + appendPadding(untilNextLineStartsAt: 262_144 - 1 - namePrefixLength) + appendLine(#"{"name":"東京"}"#) + appendPadding(untilNextLineStartsAt: JSONLineReader.defaultChunkSize - 1 - namePrefixLength) + appendLine(#"{"name":"東京","city":"Hà Nội"}"#) + try #require(bytes[262_144] & 0xC0 == 0x80) + try #require(bytes[JSONLineReader.defaultChunkSize] & 0xC0 == 0x80) + + let fields = try detect(bytes, fileExtension: "ndjson") + #expect(fields.map(\.name) == ["city", "name", "pad"]) + #expect(try field("name", in: fields).sampleValue == "東京") + #expect(try field("city", in: fields).sampleValue == "Hà Nội") + } + + @Test("A JSON Lines file with CRLF line endings is detected") + func crlfLineEndings() throws { + let fields = try detect(Data("{\"a\":1}\r\n{\"a\":2,\"b\":\"x\"}\r\n".utf8), fileExtension: "jsonl") + #expect(fields.map(\.name) == ["a", "b"]) + #expect(try field("a", in: fields).inferredType == .integer) + } + + @Test("A string holding U+2028 stays one row") + func lineSeparatorInsideAString() throws { + let fields = try detectLines(["{\"note\":\"a\u{2028}b\"}"]) + #expect(try field("note", in: fields).sampleValue == "a\u{2028}b") + } + + @Test("Unreadable and non-object lines are passed over") + func unreadableLinesArePassedOver() throws { + let fields = try detectLines([#"{"a":1}"#, "{ this is not json", "[1, 2]", "", #"{"b":2}"#]) + #expect(fields.map(\.name) == ["a", "b"]) + } + + @Test("A field that is only ever null is detected as text with no sample") + func nullOnlyField() throws { + let fields = try detectLines([#"{"a":null}"#, #"{"a":null}"#]) + #expect(try field("a", in: fields).inferredType == .text) + #expect(try field("a", in: fields).sampleValue == nil) + } + + @Test("Detection stops once its task is cancelled", arguments: ["ndjson", "json"]) + func cancelledDetectionStops(fileExtension: String) async throws { + let contents = fileExtension == "json" ? #"[{"a":1}]"# : "{\"a\":1}\n" + let url = try write(Data(contents.utf8), fileExtension: fileExtension) + defer { try? FileManager.default.removeItem(at: url) } + let detection = Task.detached { + withUnsafeCurrentTask { $0?.cancel() } + return try JSONImportParsing.detectFields(at: url, targetTable: nil) + } + await #expect(throws: CancellationError.self) { + _ = try await detection.value + } + } +} diff --git a/TableProTests/Plugins/JSONImportPluginTests.swift b/TableProTests/Plugins/JSONImportPluginTests.swift index 1d61e73eb1..759f9689f7 100644 --- a/TableProTests/Plugins/JSONImportPluginTests.swift +++ b/TableProTests/Plugins/JSONImportPluginTests.swift @@ -8,9 +8,9 @@ import TableProPluginKit import Testing struct JSONImportPluginTests { - private func object(_ json: String) throws -> [String: Any] { + private func object(_ json: String) throws -> NSDictionary { let parsed = try JSONSerialization.jsonObject(with: Data(json.utf8)) - return try #require(parsed as? [String: Any]) + return try #require(parsed as? NSDictionary) } private func anyValue(_ json: String) throws -> Any { @@ -110,7 +110,7 @@ struct JSONImportPluginTests { @Test("A JSON object line parses to a row") func testNdjsonLine() throws { - let row = try JSONImportParsing.parseRow(fromLine: #"{"id":1,"name":"x"}"#) + let row = try #require(try JSONImportParsing.parseRow(fromLine: Data(#"{"id":1,"name":"x"}"#.utf8))) #expect(row["id"] == .text("1")) #expect(row["name"] == .text("x")) } @@ -118,7 +118,27 @@ struct JSONImportPluginTests { @Test("A non-object line throws") func testNdjsonNonObjectThrows() { #expect(throws: PluginImportError.self) { - _ = try JSONImportParsing.parseRow(fromLine: "[1, 2, 3]") + _ = try JSONImportParsing.parseRow(fromLine: Data("[1, 2, 3]".utf8)) + } + } + + @Test("A line of JSON whitespace is blank, not an error") + func testNdjsonBlankLine() throws { + #expect(try JSONImportParsing.parseRow(fromLine: Data()) == nil) + #expect(try JSONImportParsing.parseRow(fromLine: Data(" \t\r".utf8)) == nil) + } + + @Test("A line ending in a carriage return parses") + func testNdjsonCarriageReturnLine() throws { + let row = try #require(try JSONImportParsing.parseRow(fromLine: Data("{\"id\":1}\r".utf8))) + #expect(row["id"] == .text("1")) + } + + @Test("A line whose bytes are not UTF-8 throws rather than importing replacement characters") + func testNdjsonInvalidUTF8Throws() { + let line = Data(#"{"name":""#.utf8) + Data([0xFF]) + Data(#""}"#.utf8) + #expect(throws: (any Error).self) { + _ = try JSONImportParsing.parseRow(fromLine: line) } } @@ -142,41 +162,54 @@ struct JSONImportPluginTests { try #require(try JSONSerialization.jsonObject(with: Data(json.utf8)) as? [Any]) } + private func inferredType(_ json: String) throws -> PluginImportFieldType { + var kinds = JSONValueKinds() + for value in try array(json) { + kinds.add(JSONValueKind(of: value)) + } + return kinds.inferredType + } + @Test("Inference: all integers") func testInferInteger() throws { - #expect(JSONImportParsing.inferType(from: try array("[1, 2, 3]")) == .integer) + #expect(try inferredType("[1, 2, 3]") == .integer) } @Test("Inference: any decimal makes the field real") func testInferReal() throws { - #expect(JSONImportParsing.inferType(from: try array("[1, 2.5, 3]")) == .real) + #expect(try inferredType("[1, 2.5, 3]") == .real) } @Test("Inference: all booleans") func testInferBoolean() throws { - #expect(JSONImportParsing.inferType(from: try array("[true, false]")) == .boolean) + #expect(try inferredType("[true, false]") == .boolean) } @Test("Inference: all-nested values are json") func testInferJSON() throws { - #expect(JSONImportParsing.inferType(from: try array(#"[{"a":1}, [1,2]]"#)) == .json) + #expect(try inferredType(#"[{"a":1}, [1,2]]"#) == .json) } @Test("Inference: mixed types fall back to text") func testInferText() throws { - #expect(JSONImportParsing.inferType(from: try array(#"["a", 1]"#)) == .text) + #expect(try inferredType(#"["a", 1]"#) == .text) + } + + @Test("Inference: a value after the type settles on text keeps it text") + func testInferTextIsFinal() throws { + #expect(try inferredType(#"["a", 1, true, {"k":1}]"#) == .text) } @Test("Inference: empty values are text") - func testInferEmpty() { - #expect(JSONImportParsing.inferType(from: []) == .text) + func testInferEmpty() throws { + #expect(try inferredType("[]") == .text) } @Test("detectFields reports sorted fields with inferred types and a sample") func testDetectFields() throws { let raw = #"[{"id":1,"name":"a","active":true},{"id":2,"name":"b","active":false}]"# - let rows = try #require(try JSONSerialization.jsonObject(with: Data(raw.utf8)) as? [[String: Any]]) - let fields = JSONImportParsing.detectFields(in: rows) + let rows = try #require(try JSONSerialization.jsonObject(with: Data(raw.utf8)) as? [NSDictionary]) + let fields = try JSONImportParsing.detectFields(in: rows) #expect(fields.map(\.name) == ["active", "id", "name"]) #expect(fields.first { $0.name == "id" }?.inferredType == .integer) #expect(fields.first { $0.name == "active" }?.inferredType == .boolean) diff --git a/TableProTests/Plugins/JSONImportSkipTests.swift b/TableProTests/Plugins/JSONImportSkipTests.swift index b7d512746f..b2ccbd11ca 100644 --- a/TableProTests/Plugins/JSONImportSkipTests.swift +++ b/TableProTests/Plugins/JSONImportSkipTests.swift @@ -11,11 +11,11 @@ private final class CountingSink: PluginImportDataSink, @unchecked Sendable { let databaseTypeId = "mock" let targetTable: String? = "people" - private(set) var insertedRows = 0 + private(set) var rows: [[String: PluginCellValue]] = [] func execute(statement: String) async throws {} - func insertRow(_ values: [String: PluginCellValue]) async throws { insertedRows += 1 } - func insertRows(_ rows: [[String: PluginCellValue]]) async throws { insertedRows += rows.count } + func insertRow(_ values: [String: PluginCellValue]) async throws { rows.append(values) } + func insertRows(_ rows: [[String: PluginCellValue]]) async throws { self.rows.append(contentsOf: rows) } func deleteAllRowsFromTargetTable() async throws {} func beginTransaction() async throws {} func commitTransaction() async throws {} @@ -45,18 +45,34 @@ private final class FileSource: PluginImportSource, @unchecked Sendable { /// instances in flight at once read each other's error-handling mode. @Suite("JSON import skips unreadable lines", .serialized) struct JSONImportSkipTests { - private func writeNDJSON(_ lines: [String]) throws -> URL { + private func writeNDJSON(_ contents: Data) throws -> URL { let url = FileManager.default.temporaryDirectory .appendingPathComponent("json-import-\(UUID().uuidString).ndjson") - try lines.joined(separator: "\n").write(to: url, atomically: true, encoding: .utf8) + try contents.write(to: url) return url } private func runImport( _ lines: [String], - errorHandling: ImportErrorHandling + errorHandling: ImportErrorHandling, + sink: CountingSink = CountingSink(), + progress: Progress = Progress() ) async throws -> Result { - let url = try writeNDJSON(lines) + try await runImport( + contents: Data(lines.joined(separator: "\n").utf8), + errorHandling: errorHandling, + sink: sink, + progress: progress + ) + } + + private func runImport( + contents: Data, + errorHandling: ImportErrorHandling, + sink: CountingSink = CountingSink(), + progress: Progress = Progress() + ) async throws -> Result { + let url = try writeNDJSON(contents) defer { try? FileManager.default.removeItem(at: url) } /// `settings` persists through plugin storage, so a test that writes it changes the @@ -71,8 +87,8 @@ struct JSONImportSkipTests { do { let result = try await plugin.performImport( source: FileSource(url: url), - sink: CountingSink(), - progress: PluginImportProgress(progress: Progress()) + sink: sink, + progress: PluginImportProgress(progress: progress) ) return .success(result) } catch { @@ -166,4 +182,108 @@ struct JSONImportSkipTests { #expect(result.skippedStatements == 0) #expect(result.errors.isEmpty) } + + /// JSON allows U+2028, U+2029 and U+0085 unescaped inside a string. Reading the file with + /// `URL.lines` ended a line at each of them and failed both halves. + @Test("A string holding a Unicode line separator imports as one row") + func unicodeSeparatorInsideAStringImportsWhole() async throws { + let sink = CountingSink() + let outcome = try await runImport( + ["{\"note\":\"a\u{2028}b\u{2029}c\u{0085}d\"}", #"{"note": "e"}"#], + errorHandling: .skipAndContinue, + sink: sink + ) + guard case .success(let result) = outcome else { + Issue.record("A valid file must import: \(outcome)") + return + } + #expect(result.executedStatements == 2) + #expect(result.skippedStatements == 0) + let first = try #require(sink.rows.first) + #expect(first["note"] == .text("a\u{2028}b\u{2029}c\u{0085}d")) + } + + @Test("Lines ending in CRLF import every row") + func crlfLinesImport() async throws { + let outcome = try await runImport( + contents: Data("{\"a\":1}\r\n\r\n{\"a\":2}\r\n".utf8), + errorHandling: .stopAndRollback + ) + guard case .success(let result) = outcome else { + Issue.record("A CRLF file must import: \(outcome)") + return + } + #expect(result.executedStatements == 2) + } + + /// The same bytes in a `.json` file fail the whole parse. A JSON Lines file used to have the + /// bad byte swapped for U+FFFD and imported as if nothing were wrong. + @Test("A line that is not UTF-8 is reported rather than imported with replacement characters") + func invalidUTF8LineIsReported() async throws { + var contents = Data("{\"name\": \"Ada\"}\n{\"name\": \"".utf8) + contents.append(0xFF) + contents.append(Data("\"}\n{\"name\": \"Grace\"}".utf8)) + let sink = CountingSink() + let outcome = try await runImport(contents: contents, errorHandling: .skipAndContinue, sink: sink) + guard case .success(let result) = outcome else { + Issue.record("Skip and Continue must not abort the import: \(outcome)") + return + } + #expect(result.executedStatements == 2) + #expect(result.skippedStatements == 1) + #expect(result.errors.contains { $0.line == 2 }) + #expect(sink.rows.compactMap { $0["name"] } == [.text("Ada"), .text("Grace")]) + } + + /// `URL.lines` reads through `FileHandle.AsyncBytes`, which Foundation serves from one queue for + /// the whole process. A reader parked on a quiet pipe, as the Copilot language server's is, + /// held that queue and the import waited behind it. + @Test("An import finishes while another reader in the process waits on a quiet pipe") + func importIgnoresABlockedAsyncBytesReader() async throws { + let pipe = Pipe() + let blocker = Task { + for try await _ in pipe.fileHandleForReading.bytes {} + } + try await Task.sleep(nanoseconds: 100_000_000) + + let finished = await withTaskGroup(of: Bool.self) { group in + group.addTask { + let outcome = try? await self.runImport( + [#"{"name": "Ada"}"#, #"{"name": "Grace"}"#], + errorHandling: .stopAndRollback + ) + guard case .success(let result) = outcome else { return false } + return result.executedStatements == 2 + } + group.addTask { + try? await Task.sleep(nanoseconds: 10_000_000_000) + return false + } + let first = await group.next() ?? false + try? pipe.fileHandleForWriting.close() + group.cancelAll() + return first + } + blocker.cancel() + + #expect(finished) + } + + /// The runner checks for a stop between batches, and a batch used to end only once it held 500 + /// rows. A long run of lines with no row in them was read to the end of the file first. + @Test("A stopped import stops within a batch of lines even when no line holds a row") + func stopReachesARunOfUnreadableLines() async throws { + let progress = Progress() + progress.cancel() + let outcome = try await runImport( + Array(repeating: "{ this is not json", count: 5_000), + errorHandling: .skipAndContinue, + progress: progress + ) + guard case .failure(let error) = outcome else { + Issue.record("A stopped import must not run to the end of the file: \(outcome)") + return + } + #expect(error is PluginImportCancellationError) + } } diff --git a/TableProTests/Plugins/JSONLineReaderTests.swift b/TableProTests/Plugins/JSONLineReaderTests.swift new file mode 100644 index 0000000000..fa8860df60 --- /dev/null +++ b/TableProTests/Plugins/JSONLineReaderTests.swift @@ -0,0 +1,106 @@ +// +// JSONLineReaderTests.swift +// TableProTests +// + +import Foundation +import Testing + +struct JSONLineReaderTests { + private func write(_ bytes: Data) throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("json-line-reader-\(UUID().uuidString).ndjson") + try bytes.write(to: url) + return url + } + + private func readLines(of bytes: Data, chunkSize: Int) throws -> [Data] { + let url = try write(bytes) + defer { try? FileManager.default.removeItem(at: url) } + var reader = try JSONLineReader(url: url, chunkSize: chunkSize) + defer { reader.close() } + var lines: [Data] = [] + while let line = try reader.next() { + lines.append(Data(line)) + } + return lines + } + + private func text(_ lines: [Data]) -> [String?] { + lines.map { String(data: $0, encoding: .utf8) } + } + + @Test("Lines split on the newline byte, with and without a final newline") + func splitsOnNewline() throws { + for chunkSize in [1, 3, 64, JSONLineReader.defaultChunkSize] { + #expect(try text(readLines(of: Data("a\nbb\nccc\n".utf8), chunkSize: chunkSize)) == ["a", "bb", "ccc"]) + #expect(try text(readLines(of: Data("a\nbb\nccc".utf8), chunkSize: chunkSize)) == ["a", "bb", "ccc"]) + } + } + + @Test("An empty file has no lines and a blank line is still a line") + func emptyAndBlankLines() throws { + #expect(try readLines(of: Data(), chunkSize: 4).isEmpty) + #expect(try text(readLines(of: Data("a\n\n\nb\n".utf8), chunkSize: 2)) == ["a", "", "", "b"]) + } + + @Test("Line numbers count blank lines") + func lineNumbersCountBlankLines() throws { + let url = try write(Data("a\n\nb\n".utf8)) + defer { try? FileManager.default.removeItem(at: url) } + var reader = try JSONLineReader(url: url, chunkSize: 2) + defer { reader.close() } + var numbered: [Int: String] = [:] + while let line = try reader.next() { + numbered[reader.lineNumber] = String(data: Data(line), encoding: .utf8) + } + #expect(numbered == [1: "a", 2: "", 3: "b"]) + } + + @Test("A carriage return before the newline stays part of the line") + func carriageReturnStaysInTheLine() throws { + #expect(try text(readLines(of: Data("a\r\nb\r\n".utf8), chunkSize: 3)) == ["a\r", "b\r"]) + } + + /// JSON allows these unescaped inside a string. `URL.lines` ends a line at each of them, + /// which cut a valid row in two. + @Test("U+2028, U+2029 and U+0085 do not end a line") + func unicodeSeparatorsDoNotEndALine() throws { + let line = "{\"a\":\"x\u{2028}y\u{2029}z\u{0085}w\"}" + #expect(try text(readLines(of: Data("\(line)\n{}\n".utf8), chunkSize: 5)) == [line, "{}"]) + } + + @Test("A multi-byte character that straddles a chunk boundary reads back whole") + func multiByteCharacterAcrossChunks() throws { + let line = "{\"tên\":\"Nguyễn 東京\"}" + let bytes = Data("\(line)\n\(line)\n".utf8) + for chunkSize in 1...12 { + #expect(try text(readLines(of: bytes, chunkSize: chunkSize)) == [line, line]) + } + } + + @Test("A line many chunks long reads back whole") + func lineLongerThanManyChunks() throws { + let long = String(repeating: "東", count: 10_000) + let lines = try text(readLines(of: Data("\(long)\nshort\n".utf8), chunkSize: 7)) + #expect(lines == [long, "short"]) + } + + /// Detection checked for a stop between lines only, so closing the sheet over a file with no + /// newline read the whole file into one line first. + @Test("A stop is checked before every chunk, so a line with no end is abandoned") + func stopIsCheckedBeforeEveryChunk() throws { + let url = try write(Data(String(repeating: "x", count: 1_000).utf8)) + defer { try? FileManager.default.removeItem(at: url) } + var checks = 0 + var reader = try JSONLineReader(url: url, chunkSize: 10) { + checks += 1 + if checks > 3 { throw CancellationError() } + } + defer { reader.close() } + #expect(throws: CancellationError.self) { + _ = try reader.next() + } + #expect(checks == 4) + } +} diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index 0817ca5eb0..18de025a9e 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -323,6 +323,8 @@ The sheet accepts an array of objects `[{…}, {…}]`, newline-delimited JSON s - **Existing table**: map each JSON field to a column. Fields auto-match by name; switch one off to skip it. A column with no matching field keeps its default or NULL. The list holds tables and partitioned tables, and the server routes each row to the right partition. - **New table**: the name field opens on a name derived from the file, already selected, so one keystroke replaces it. Review the inferred columns underneath. Name, type, primary key, nullable flag, and default are all editable before the table is created. +Both lists come from every row in the file. A key that only the last rows carry is still offered, and an inferred type fits every value the field holds, so a field that is a number in all rows but one is typed as text. + The proposed name drops the extension, turns spaces and punctuation into underscores, and lowercases the result. Letters from any script are kept as they are. On Oracle the name comes through in upper case instead, and is cut to 30 bytes rather than 63. A name an existing table or view already holds gains a numeric suffix, so re-importing `users.csv` next to a `users` table proposes `users_2`. Whatever you type over it is held to the same rules: a name already taken, one longer than the engine allows, or one starting with a prefix the engine keeps for itself is reported in the sheet, with **Import** off until it changes. Rows insert through parameterized statements, so a JSON value is never concatenated into SQL. Nested objects and arrays are stored as JSON text. In a new PostgreSQL table their column is `jsonb`, or `json` on 9.2 and 9.3, and `text` on 9.1, which has no JSON type. diff --git a/project.yml b/project.yml index 50aa44a369..33d21160b6 100644 --- a/project.yml +++ b/project.yml @@ -533,10 +533,12 @@ targets: - Plugins/MarkdownExportPlugin/MarkdownExportModels.swift - Plugins/ParquetExportPlugin/ParquetExportModels.swift - Plugins/ParquetExportPlugin/ParquetTypeMapper.swift + - Plugins/JSONImportPlugin/JSONFieldSurvey.swift - Plugins/JSONImportPlugin/JSONImportOptions.swift - Plugins/JSONImportPlugin/JSONImportOptionsView.swift - Plugins/JSONImportPlugin/JSONImportParsing.swift - Plugins/JSONImportPlugin/JSONImportPlugin.swift + - Plugins/JSONImportPlugin/JSONLineReader.swift - Plugins/MSSQLDriverPlugin/MSSQLObjectQueries.swift - Plugins/MSSQLDriverPlugin/MSSQLTypeQueries.swift - Plugins/MSSQLDriverPlugin/MSSQLTypeDefinition.swift From d3e39abc4bfbc8724db4d98ae6803c723ed63c97 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Tue, 29 Sep 2026 22:01:30 +0700 Subject: [PATCH 2/3] fix(plugins): re-detect import fields after an option change and reuse JSON Lines fields across tables --- .../JSONFieldDetectionCache.swift | 52 +++++++ .../JSONImportPlugin/JSONImportParsing.swift | 4 + .../JSONImportPlugin/JSONImportPlugin.swift | 44 +++--- .../JSONImportPlugin/JSONLineBatches.swift | 66 ++++++++ .../Services/Export/ImportFieldList.swift | 41 +++++ TablePro/Resources/Localizable.xcstrings | 3 + .../Import/ImportPluginObservation.swift | 29 ++++ TablePro/Views/Import/RowImportSheet.swift | 146 +++++++++--------- .../Core/Services/ImportFieldListTests.swift | 79 ++++++++++ .../JSONFieldDetectionCacheTests.swift | 107 +++++++++++++ .../JSONImportFieldDetectionTests.swift | 28 ++++ .../Plugins/JSONImportSkipTests.swift | 29 +--- .../Plugins/JSONLineBatchesTests.swift | 78 ++++++++++ .../Views/ImportPluginObservationTests.swift | 54 +++++++ project.yml | 2 + 15 files changed, 642 insertions(+), 120 deletions(-) create mode 100644 Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift create mode 100644 Plugins/JSONImportPlugin/JSONLineBatches.swift create mode 100644 TablePro/Core/Services/Export/ImportFieldList.swift create mode 100644 TablePro/Views/Import/ImportPluginObservation.swift create mode 100644 TableProTests/Core/Services/ImportFieldListTests.swift create mode 100644 TableProTests/Plugins/JSONFieldDetectionCacheTests.swift create mode 100644 TableProTests/Plugins/JSONLineBatchesTests.swift create mode 100644 TableProTests/Views/ImportPluginObservationTests.swift diff --git a/Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift b/Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift new file mode 100644 index 0000000000..ab5ac80531 --- /dev/null +++ b/Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift @@ -0,0 +1,52 @@ +// +// JSONFieldDetectionCache.swift +// JSONImportPlugin +// + +import Foundation +import os +import TableProPluginKit + +/// The fields of the last JSON Lines file read, kept against that file's identity. +/// +/// A JSON Lines file's fields do not depend on the table they are matched against, and reading +/// them means reading every line. The import sheet asks again on every destination table pick, so +/// each pick used to read the whole file again. +final class JSONFieldDetectionCache: Sendable { + /// A file counts as unchanged while its path, file number, size and modification date all + /// match. The identity is taken before the read, so an edit made during one leaves a stale + /// identity behind and the next request reads the file again. It describes the file a link + /// points to, because `attributesOfItem` describes the link itself and the read follows it. + private struct FileIdentity: Equatable, Sendable { + let path: String + let fileNumber: UInt64? + let size: UInt64? + let modificationDate: Date? + + init(of url: URL) throws { + let file = url.resolvingSymlinksInPath() + let attributes = try FileManager.default.attributesOfItem(atPath: file.path) + path = file.path + fileNumber = (attributes[.systemFileNumber] as? NSNumber)?.uint64Value + size = (attributes[.size] as? NSNumber)?.uint64Value + modificationDate = attributes[.modificationDate] as? Date + } + } + + private struct Entry: Sendable { + let file: FileIdentity + let fields: [PluginImportField] + } + + private let lastEntry = OSAllocatedUnfairLock(initialState: nil) + + func fields(at url: URL, detect: () throws -> [PluginImportField]) throws -> [PluginImportField] { + let file = try FileIdentity(of: url) + if let cached = lastEntry.withLock({ $0?.file == file ? $0?.fields : nil }) { + return cached + } + let fields = try detect() + lastEntry.withLock { $0 = Entry(file: file, fields: fields) } + return fields + } +} diff --git a/Plugins/JSONImportPlugin/JSONImportParsing.swift b/Plugins/JSONImportPlugin/JSONImportParsing.swift index f0fb4ee9d9..f7722e94d9 100644 --- a/Plugins/JSONImportPlugin/JSONImportParsing.swift +++ b/Plugins/JSONImportPlugin/JSONImportParsing.swift @@ -113,6 +113,10 @@ enum JSONImportParsing { guard isLineDelimited(url) else { return try detectFields(in: try parseRows(at: url, targetTable: targetTable)) } + return try detectFields(inLinesAt: url) + } + + static func detectFields(inLinesAt url: URL) throws -> [PluginImportField] { var survey = JSONFieldSurvey() var lines = try JSONLineReader(url: url) defer { lines.close() } diff --git a/Plugins/JSONImportPlugin/JSONImportPlugin.swift b/Plugins/JSONImportPlugin/JSONImportPlugin.swift index 5d4f5323a3..862385b769 100644 --- a/Plugins/JSONImportPlugin/JSONImportPlugin.swift +++ b/Plugins/JSONImportPlugin/JSONImportPlugin.swift @@ -42,6 +42,8 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug /// truncated list must not also under-report how much of the file was left out. private static let maxRecordedErrors = 1_000 + private let lineDelimitedFields = JSONFieldDetectionCache() + func performImport( source: any PluginImportSource, sink: any PluginImportDataSink, @@ -60,35 +62,20 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug var unreadableLineCount = 0 if JSONImportParsing.isLineDelimited(url) { progress.setEstimatedTotal(max(1, Int(source.fileSizeBytes() / 256))) - var lines = try JSONLineReader(url: url, checkCancellation: progress.checkCancellation) - defer { lines.close() } - let skipsErrors = settings.errorHandling == .skipAndContinue + var batches = JSONLineBatches( + lines: try JSONLineReader(url: url, checkCancellation: progress.checkCancellation), + linesPerBatch: Self.batchSize, + skipsUnreadableLines: settings.errorHandling == .skipAndContinue, + maxRecordedErrors: Self.maxRecordedErrors + ) + defer { batches.close() } outcome = try await RowImportRunner.run( configuration: configuration, sink: sink, progress: progress ) { - var batch: [RowImportRunner.Entry] = [] - var linesRead = 0 - while linesRead < Self.batchSize, let line = try lines.next() { - linesRead += 1 - let lineNumber = lines.lineNumber - do { - let row = try autoreleasepool { try JSONImportParsing.parseRow(fromLine: line) } - guard let row, !row.isEmpty else { continue } - batch.append((lineNumber, row)) - } catch { - guard skipsErrors else { throw error } - unreadableLineCount += 1 - if unreadableLines.count < Self.maxRecordedErrors { - unreadableLines.append(.init( - statement: "row \(lineNumber)", - line: lineNumber, - errorMessage: error.localizedDescription - )) - } - } - } - return linesRead == 0 ? nil : batch + try batches.next() } + unreadableLines = batches.unreadableLines + unreadableLineCount = batches.unreadableLineCount } else { let rawRows = try JSONImportParsing.parseRows(at: url, targetTable: sink.targetTable) progress.setEstimatedTotal(rawRows.count) @@ -119,6 +106,11 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug // MARK: - Source introspection func detectSourceFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { - try JSONImportParsing.detectFields(at: url, targetTable: targetTable) + guard JSONImportParsing.isLineDelimited(url) else { + return try JSONImportParsing.detectFields(at: url, targetTable: targetTable) + } + return try lineDelimitedFields.fields(at: url) { + try JSONImportParsing.detectFields(inLinesAt: url) + } } } diff --git a/Plugins/JSONImportPlugin/JSONLineBatches.swift b/Plugins/JSONImportPlugin/JSONLineBatches.swift new file mode 100644 index 0000000000..b377a77c3c --- /dev/null +++ b/Plugins/JSONImportPlugin/JSONLineBatches.swift @@ -0,0 +1,66 @@ +// +// JSONLineBatches.swift +// JSONImportPlugin +// + +import Foundation +import TableProPluginKit + +/// Hands the rows of a JSON Lines file to `RowImportRunner` one batch at a time, and keeps the +/// lines Skip and Continue passed over. +/// +/// A batch ends after a set number of lines, not rows. The runner checks for a stop between +/// batches, so a batch that waited for its rows read on through a run of lines holding none, up +/// to a whole chunk of the file, before a stop was seen. +struct JSONLineBatches { + private var lines: JSONLineReader + private let linesPerBatch: Int + private let skipsUnreadableLines: Bool + private let maxRecordedErrors: Int + + private(set) var unreadableLines: [PluginImportResult.ImportStatementError] = [] + + /// Every unreadable line, counted past the end of the capped `unreadableLines` list. + private(set) var unreadableLineCount = 0 + + var linesRead: Int { lines.lineNumber } + + init(lines: JSONLineReader, linesPerBatch: Int, skipsUnreadableLines: Bool, maxRecordedErrors: Int) { + self.lines = lines + self.linesPerBatch = max(1, linesPerBatch) + self.skipsUnreadableLines = skipsUnreadableLines + self.maxRecordedErrors = maxRecordedErrors + } + + mutating func next() throws -> [RowImportRunner.Entry]? { + var batch: [RowImportRunner.Entry] = [] + var linesInBatch = 0 + while linesInBatch < linesPerBatch, let line = try lines.next() { + linesInBatch += 1 + let lineNumber = lines.lineNumber + do { + let row = try autoreleasepool { try JSONImportParsing.parseRow(fromLine: line) } + guard let row, !row.isEmpty else { continue } + batch.append((lineNumber, row)) + } catch { + guard skipsUnreadableLines else { throw error } + recordUnreadableLine(lineNumber, error: error) + } + } + return linesInBatch == 0 ? nil : batch + } + + func close() { + lines.close() + } + + private mutating func recordUnreadableLine(_ lineNumber: Int, error: any Error) { + unreadableLineCount += 1 + guard unreadableLines.count < maxRecordedErrors else { return } + unreadableLines.append(.init( + statement: "row \(lineNumber)", + line: lineNumber, + errorMessage: error.localizedDescription + )) + } +} diff --git a/TablePro/Core/Services/Export/ImportFieldList.swift b/TablePro/Core/Services/Export/ImportFieldList.swift new file mode 100644 index 0000000000..ecc5eb2615 --- /dev/null +++ b/TablePro/Core/Services/Export/ImportFieldList.swift @@ -0,0 +1,41 @@ +// +// ImportFieldList.swift +// TablePro +// + +import Foundation + +/// What a field list in the import sheet is read for: the table its fields are matched against, +/// if any, the plugin options that shape detection, and the Try Again it answers. +struct ImportFieldDetectionRequest: Hashable { + let targetTable: String? + let detectionSignature: String + let attempt: Int +} + +/// A field list together with the request it was read for. +/// +/// The rows answer that request and no other, so a change to any part of it, an option that +/// shapes detection included, reads the file again. The request travels with the rows because a +/// separate "loaded" flag, cleared from `onChange`, was cleared after the `.task(id:)` that read +/// it: SwiftUI ran the restarted task first, and the fields of the old options stayed on screen. +struct ImportFieldList { + var rows: [Row] = [] + private(set) var readFor: ImportFieldDetectionRequest? + + func needsRead(for request: ImportFieldDetectionRequest) -> Bool { + readFor != request + } + + /// Empties the list ahead of a read for another request. The request goes with the rows, or a + /// return to it before the read finished would find it answered and show an empty list. + mutating func discard() { + rows = [] + readFor = nil + } + + mutating func finishRead(_ rows: [Row], for request: ImportFieldDetectionRequest) { + self.rows = rows + readFor = request + } +} diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 4732c63b49..df38622157 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -122588,6 +122588,9 @@ }, "Reading Tables" : { + }, + "Reading the file…" : { + }, "Reading the source…" : { diff --git a/TablePro/Views/Import/ImportPluginObservation.swift b/TablePro/Views/Import/ImportPluginObservation.swift new file mode 100644 index 0000000000..1e527fa34e --- /dev/null +++ b/TablePro/Views/Import/ImportPluginObservation.swift @@ -0,0 +1,29 @@ +// +// ImportPluginObservation.swift +// TablePro +// + +import Combine +import Foundation +import TableProPluginKit + +/// Relays an import plugin's own changes to the import sheet. +/// +/// The sheet shows the plugin's options view, which observes the plugin, but the sheet reaches the +/// plugin through `PluginManager`, and nothing `PluginManager` publishes changes when an option +/// does. Measured: a new option redrew the options view and not the sheet, so the sheet never read +/// the new `fieldDetectionSignature` and kept the fields read for the old options. +@MainActor +internal final class ImportPluginObservation: ObservableObject { + private var subscription: AnyCancellable? + + internal init(plugin: (any ImportFormatPlugin)?) { + guard let observable = plugin as? any ObservableObject else { return } + subscription = Self.changes(of: observable) + .sink { [weak self] in self?.objectWillChange.send() } + } + + private static func changes(of object: some ObservableObject) -> AnyPublisher { + object.objectWillChange.map { _ in }.eraseToAnyPublisher() + } +} diff --git a/TablePro/Views/Import/RowImportSheet.swift b/TablePro/Views/Import/RowImportSheet.swift index 674c68257d..cb4c6b9d5b 100644 --- a/TablePro/Views/Import/RowImportSheet.swift +++ b/TablePro/Views/Import/RowImportSheet.swift @@ -28,14 +28,11 @@ struct RowImportSheet: View { case newTable } - /// What the field list on screen is read for. Detection reads the whole file, so it runs only - /// for the destination on screen, and a request that changes or a sheet that closes cancels the - /// read still running for the old one. - private struct FieldDetectionRequest: Hashable { + /// Detection reads the whole file, so it runs only for the destination on screen, and a request + /// that changes or a sheet that closes cancels the read still running for the old one. + private struct FieldDetectionTask: Hashable { let destination: Destination - let targetTable: String? - let detectionSignature: String - let attempt: Int + let request: ImportFieldDetectionRequest } private struct FieldMapping: Identifiable { @@ -76,21 +73,24 @@ struct RowImportSheet: View { @State private var isLoadingTables = false @State private var selectedTargetTable: String? @State private var targetColumns: [String] = [] - @State private var mappings: [FieldMapping] = [] + + /// Each list keeps the request it was read for, so switching destination and back keeps the + /// user's edits rather than reading the file again and replacing them with fresh matches. + @State private var mappings = ImportFieldList() @State private var newTableName: String = "" /// The last name this sheet proposed, so a second pass can tell its own guess from what /// the user typed over it. @State private var proposedTableName: String = "" - @State private var newColumns: [NewColumn] = [] - @State private var newColumnsLoaded = false - /// The table `mappings` were built for. Switching destination and back keeps the user's edits - /// rather than reading the file again and replacing them with fresh matches. - @State private var mappedTable: String? + @State private var newColumns = ImportFieldList() @State private var isLoadingContext = false @State private var loadError: String? @State private var detectionAttempt = 0 + /// Redraws the sheet when an option changes, so the field detection request sees the plugin's + /// new `fieldDetectionSignature`. + @StateObject private var pluginObservation: ImportPluginObservation + /// Moving focus here also selects the whole proposed name, measured rather than assumed: /// SwiftUI hands the field editor a full selection when `@FocusState` lands on text already in /// place, both on appear and when the field is revealed by the destination picker. So the first @@ -122,6 +122,16 @@ struct RowImportSheet: View { /// torn down in the same transaction, and AppKit ends a sheet's children with it (#2314). @State private var hostWindow: NSWindow? + init(isPresented: Binding, connection: DatabaseConnection, fileURL: URL, formatId: String) { + _isPresented = isPresented + self.connection = connection + self.fileURL = fileURL + self.formatId = formatId + _pluginObservation = StateObject( + wrappedValue: ImportPluginObservation(plugin: PluginManager.shared.importPlugin(forFormat: formatId)) + ) + } + // MARK: - Derived catalog state /// Tables alone, because they are the only objects the existing-table branch can insert into. @@ -181,23 +191,14 @@ struct RowImportSheet: View { suggestNewTableName() await loadTables() } - .task(id: fieldDetectionRequest) { - await loadFields(for: fieldDetectionRequest) + .task(id: fieldDetectionTask) { + await loadFields(for: fieldDetectionTask) } .onChange(of: destination) { newValue in guard newValue == .newTable else { return } suggestNewTableName() newTableNameFocused = true } - .onChange(of: selectedTargetTable) { _ in - mappings = [] - targetColumns = [] - mappedTable = nil - } - .onChange(of: currentPlugin?.fieldDetectionSignature) { _ in - newColumnsLoaded = false - mappedTable = nil - } .onDisappear { importTask?.cancel() if !importSucceeded { settingsSnapshot?.restore() } @@ -247,9 +248,6 @@ struct RowImportSheet: View { .foregroundStyle(.secondary) } Spacer() - if isLoadingContext { - ProgressView().controlSize(.small) - } } } @@ -356,18 +354,20 @@ struct RowImportSheet: View { private var contentArea: some View { if let loadError { unreadableFile(reason: loadError) + } else if isLoadingContext { + readingFile } else { switch destination { case .existingTable: if selectedTargetTable == nil { placeholder("Choose a destination table to map fields.") - } else if mappings.isEmpty { + } else if mappings.rows.isEmpty { placeholder("No fields found in the file.") } else { mappingTable } case .newTable: - if newColumns.isEmpty { + if newColumns.rows.isEmpty { placeholder("No columns found in the file.") } else { newColumnsTable @@ -376,6 +376,13 @@ struct RowImportSheet: View { } } + /// A read takes seconds on a large file, and until it ends the list is empty, which the + /// placeholders below would report as a file with no fields in it. + private var readingFile: some View { + ProgressView(String(localized: "Reading the file…")) + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + /// A file the plugin could not read is a failure, not an empty result. Showing the parser's /// message as grey placeholder text left the sheet with nothing to press but Cancel. private func unreadableFile(reason: String) -> some View { @@ -392,10 +399,6 @@ struct RowImportSheet: View { private func retryLoad() { loadError = nil - newColumnsLoaded = false - newColumns = [] - mappings = [] - mappedTable = nil detectionAttempt += 1 } @@ -433,7 +436,7 @@ struct RowImportSheet: View { ScrollView { VStack(spacing: 6) { - ForEach(mappings) { row in + ForEach(mappings.rows) { row in mappingRow(row) } } @@ -504,7 +507,7 @@ struct RowImportSheet: View { ScrollView { VStack(spacing: 6) { - ForEach(newColumns) { row in + ForEach(newColumns.rows) { row in newColumnRow(row) } } @@ -557,37 +560,37 @@ struct RowImportSheet: View { // MARK: - Bindings private func mappingBinding(_ row: FieldMapping) -> Binding { - guard let index = mappings.firstIndex(where: { $0.id == row.id }) else { + guard let index = mappings.rows.firstIndex(where: { $0.id == row.id }) else { return .constant(row) } - return $mappings[index] + return $mappings.rows[index] } private func columnBinding(_ row: NewColumn) -> Binding { - guard let index = newColumns.firstIndex(where: { $0.id == row.id }) else { + guard let index = newColumns.rows.firstIndex(where: { $0.id == row.id }) else { return .constant(row) } - return $newColumns[index] + return $newColumns.rows[index] } private var allMappingsIncluded: Binding { Binding( - get: { !mappings.isEmpty && mappings.allSatisfy(\.include) }, - set: { value in for index in mappings.indices { mappings[index].include = value } } + get: { !mappings.rows.isEmpty && mappings.rows.allSatisfy(\.include) }, + set: { value in for index in mappings.rows.indices { mappings.rows[index].include = value } } ) } private var allColumnsIncluded: Binding { Binding( - get: { !newColumns.isEmpty && newColumns.allSatisfy(\.include) }, - set: { value in for index in newColumns.indices { newColumns[index].include = value } } + get: { !newColumns.rows.isEmpty && newColumns.rows.allSatisfy(\.include) }, + set: { value in for index in newColumns.rows.indices { newColumns.rows[index].include = value } } ) } private var validationMessage: String? { switch destination { case .existingTable: - let columns = mappings.filter { $0.include }.compactMap { $0.targetColumn?.lowercased() } + let columns = mappings.rows.filter { $0.include }.compactMap { $0.targetColumn?.lowercased() } if Set(columns).count != columns.count { return String(localized: "Each column can be mapped from only one field.") } @@ -614,7 +617,7 @@ struct RowImportSheet: View { ) } } - let names = newColumns + let names = newColumns.rows .filter { $0.include } .map { $0.name.trimmingCharacters(in: .whitespaces).lowercased() } if names.contains(where: \.isEmpty) { @@ -663,10 +666,10 @@ struct RowImportSheet: View { guard !(importService?.state.isImporting ?? false), validationMessage == nil else { return false } switch destination { case .existingTable: - return selectedTargetTable != nil && mappings.contains { $0.include && $0.targetColumn != nil } + return selectedTargetTable != nil && mappings.rows.contains { $0.include && $0.targetColumn != nil } case .newTable: return !newTableName.trimmingCharacters(in: .whitespaces).isEmpty - && newColumns.contains { $0.include && !$0.name.trimmingCharacters(in: .whitespaces).isEmpty } + && newColumns.rows.contains { $0.include && !$0.name.trimmingCharacters(in: .whitespaces).isEmpty } } } @@ -726,41 +729,43 @@ struct RowImportSheet: View { newTableName = suggestion } - private var fieldDetectionRequest: FieldDetectionRequest { - FieldDetectionRequest( + private var fieldDetectionTask: FieldDetectionTask { + FieldDetectionTask( destination: destination, - targetTable: destination == .existingTable ? selectedTargetTable : nil, - detectionSignature: currentPlugin?.fieldDetectionSignature ?? "", - attempt: detectionAttempt + request: ImportFieldDetectionRequest( + targetTable: destination == .existingTable ? selectedTargetTable : nil, + detectionSignature: currentPlugin?.fieldDetectionSignature ?? "", + attempt: detectionAttempt + ) ) } /// A cancelled load leaves every piece of state to the load that replaced it, or to nobody once /// the sheet has closed. Only the current request clears the loading indicator. @MainActor - private func loadFields(for request: FieldDetectionRequest) async { + private func loadFields(for task: FieldDetectionTask) async { isLoadingContext = true loadError = nil - switch request.destination { + switch task.destination { case .newTable: - await loadNewColumns() + await loadNewColumns(for: task.request) case .existingTable: - if let table = request.targetTable { - await loadExistingContext(table: table) - } + await loadExistingContext(for: task.request) } guard !Task.isCancelled else { return } isLoadingContext = false } @MainActor - private func loadNewColumns() async { - guard !newColumnsLoaded, let plugin = currentPlugin else { return } + private func loadNewColumns(for request: ImportFieldDetectionRequest) async { + guard newColumns.needsRead(for: request) else { return } + newColumns.discard() + guard let plugin = currentPlugin else { return } do { let fields = try await ImportFieldDetection.detectFields(plugin: plugin, at: fileURL, targetTable: nil) guard !Task.isCancelled else { return } let serverVersion = DatabaseManager.shared.driver(for: connection.id)?.serverVersion - newColumns = fields.map { field in + let columns = fields.map { field in NewColumn( field: field, include: true, @@ -775,7 +780,7 @@ struct RowImportSheet: View { defaultValue: "" ) } - newColumnsLoaded = true + newColumns.finishRead(columns, for: request) } catch { guard !Task.isCancelled else { return } loadError = error.localizedDescription @@ -784,8 +789,11 @@ struct RowImportSheet: View { } @MainActor - private func loadExistingContext(table: String) async { - guard mappedTable != table, let plugin = currentPlugin, + private func loadExistingContext(for request: ImportFieldDetectionRequest) async { + guard mappings.needsRead(for: request) else { return } + mappings.discard() + targetColumns = [] + guard let table = request.targetTable, let plugin = currentPlugin, DatabaseManager.shared.browseScope(for: connection.id) != nil else { return } do { let columns = try await DatabaseManager.shared.withBrowseMetadataDriver( @@ -796,11 +804,11 @@ struct RowImportSheet: View { let fields = try await ImportFieldDetection.detectFields(plugin: plugin, at: fileURL, targetTable: table) guard !Task.isCancelled else { return } targetColumns = columns - mappings = fields.map { field in + let rows = fields.map { field in let match = columns.first { $0.caseInsensitiveCompare(field.name) == .orderedSame } return FieldMapping(field: field, include: match != nil, targetColumn: match) } - mappedTable = table + mappings.finishRead(rows, for: request) } catch { guard !Task.isCancelled else { return } loadError = error.localizedDescription @@ -840,7 +848,7 @@ struct RowImportSheet: View { private func existingMapping() -> [String: String] { var mapping: [String: String] = [:] - for entry in mappings where entry.include { + for entry in mappings.rows where entry.include { if let column = entry.targetColumn { mapping[entry.field.name] = column } @@ -850,14 +858,14 @@ struct RowImportSheet: View { private func newTableMapping() -> [String: String] { var mapping: [String: String] = [:] - for column in newColumns where column.include && !column.name.trimmingCharacters(in: .whitespaces).isEmpty { + for column in newColumns.rows where column.include && !column.name.trimmingCharacters(in: .whitespaces).isEmpty { mapping[column.field.name] = column.name } return mapping } private func newTableDefinition(tableName: String) -> PluginCreateTableDefinition? { - let included = newColumns.filter { + let included = newColumns.rows.filter { $0.include && !$0.name.trimmingCharacters(in: .whitespaces).isEmpty && !$0.type.trimmingCharacters(in: .whitespaces).isEmpty diff --git a/TableProTests/Core/Services/ImportFieldListTests.swift b/TableProTests/Core/Services/ImportFieldListTests.swift new file mode 100644 index 0000000000..9ae525023d --- /dev/null +++ b/TableProTests/Core/Services/ImportFieldListTests.swift @@ -0,0 +1,79 @@ +// +// ImportFieldListTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct ImportFieldListTests { + private func request( + table: String? = "people", + signature: String = CSVImportOptions().detectionSignature, + attempt: Int = 0 + ) -> ImportFieldDetectionRequest { + ImportFieldDetectionRequest(targetTable: table, detectionSignature: signature, attempt: attempt) + } + + private func list(readFor request: ImportFieldDetectionRequest) -> ImportFieldList { + var list = ImportFieldList() + list.finishRead(["id;name;email"], for: request) + return list + } + + /// A semicolon CSV read with a comma delimiter has one field, `id;name;email`. Picking `;` + /// changes only the detection options, and the sheet used to keep the one field because the + /// table it was matched against had not changed. + @Test("A new delimiter reads the file again for the same table") + func delimiterChangeNeedsARead() { + var options = CSVImportOptions() + options.delimiter = .comma + let fields = list(readFor: request(signature: options.detectionSignature)) + options.delimiter = .semicolon + #expect(fields.needsRead(for: request(signature: options.detectionSignature))) + } + + @Test("A new delimiter reads the file again for a new table") + func delimiterChangeNeedsAReadWithoutATable() { + var options = CSVImportOptions() + options.delimiter = .comma + let fields = list(readFor: request(table: nil, signature: options.detectionSignature)) + options.delimiter = .semicolon + #expect(fields.needsRead(for: request(table: nil, signature: options.detectionSignature))) + } + + /// Switching destination and back asks for the same request again, and a second read would + /// replace the user's edits with fresh matches. + @Test("The request a list was read for needs no second read") + func sameRequestKeepsTheList() { + let fields = list(readFor: request()) + #expect(!fields.needsRead(for: request())) + #expect(fields.rows == ["id;name;email"]) + } + + @Test("Another table reads the file again") + func otherTableNeedsARead() { + #expect(list(readFor: request(table: "people")).needsRead(for: request(table: "orders"))) + } + + @Test("Try Again reads the file again") + func retryNeedsARead() { + #expect(list(readFor: request(attempt: 0)).needsRead(for: request(attempt: 1))) + } + + @Test("A list that was never read needs a read") + func unreadListNeedsARead() { + #expect(ImportFieldList().needsRead(for: request())) + } + + /// Picking another table empties the list before its read. Going back to the first table + /// before that read ends has to read again, not find its request answered by an empty list. + @Test("A discarded list answers no request, the one it was read for included") + func discardForgetsTheRequest() { + var fields = list(readFor: request(table: "people")) + fields.discard() + #expect(fields.rows.isEmpty) + #expect(fields.needsRead(for: request(table: "people"))) + } +} diff --git a/TableProTests/Plugins/JSONFieldDetectionCacheTests.swift b/TableProTests/Plugins/JSONFieldDetectionCacheTests.swift new file mode 100644 index 0000000000..80857887c3 --- /dev/null +++ b/TableProTests/Plugins/JSONFieldDetectionCacheTests.swift @@ -0,0 +1,107 @@ +// +// JSONFieldDetectionCacheTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct JSONFieldDetectionCacheTests { + private func write(_ lines: [String]) throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("json-field-cache-\(UUID().uuidString).ndjson") + try Data(lines.map { $0 + "\n" }.joined().utf8).write(to: url) + return url + } + + private func append(_ line: String, to url: URL) throws { + let handle = try FileHandle(forWritingTo: url) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data((line + "\n").utf8)) + } + + @Test("An unchanged file is read once however often its fields are asked for") + func unchangedFileIsReadOnce() throws { + let url = try write([#"{"a":1}"#]) + defer { try? FileManager.default.removeItem(at: url) } + let cache = JSONFieldDetectionCache() + var reads = 0 + for _ in 0..<3 { + let fields = try cache.fields(at: url) { + reads += 1 + return try JSONImportParsing.detectFields(inLinesAt: url) + } + #expect(fields.map(\.name) == ["a"]) + } + #expect(reads == 1) + } + + @Test("A file changed since its last read is read again") + func changedFileIsReadAgain() throws { + let url = try write([#"{"a":1}"#]) + defer { try? FileManager.default.removeItem(at: url) } + let cache = JSONFieldDetectionCache() + _ = try cache.fields(at: url) { try JSONImportParsing.detectFields(inLinesAt: url) } + + try append(#"{"b":2}"#, to: url) + + let fields = try cache.fields(at: url) { try JSONImportParsing.detectFields(inLinesAt: url) } + #expect(fields.map(\.name) == ["a", "b"]) + } + + /// The read follows a symbolic link and file attributes do not, so an identity taken from the + /// link itself stayed the same while the file it points to changed. + @Test("A file reached through a symbolic link is read again once the file changes") + func changedFileBehindALinkIsReadAgain() throws { + let target = try write([#"{"a":1}"#]) + let link = target.deletingLastPathComponent() + .appendingPathComponent("json-field-cache-link-\(UUID().uuidString).ndjson") + try FileManager.default.createSymbolicLink(at: link, withDestinationURL: target) + defer { + try? FileManager.default.removeItem(at: link) + try? FileManager.default.removeItem(at: target) + } + let cache = JSONFieldDetectionCache() + _ = try cache.fields(at: link) { try JSONImportParsing.detectFields(inLinesAt: link) } + + try append(#"{"b":2}"#, to: target) + + let fields = try cache.fields(at: link) { try JSONImportParsing.detectFields(inLinesAt: link) } + #expect(fields.map(\.name) == ["a", "b"]) + } + + @Test("Another file is not answered with the fields of the last one") + func otherFileIsReadItself() throws { + let first = try write([#"{"a":1}"#]) + let second = try write([#"{"b":1}"#]) + defer { + try? FileManager.default.removeItem(at: first) + try? FileManager.default.removeItem(at: second) + } + let cache = JSONFieldDetectionCache() + _ = try cache.fields(at: first) { try JSONImportParsing.detectFields(inLinesAt: first) } + let fields = try cache.fields(at: second) { try JSONImportParsing.detectFields(inLinesAt: second) } + #expect(fields.map(\.name) == ["b"]) + } + + /// Closing the import sheet cancels the read. A cancelled read has no fields to keep, and the + /// next request has to read the file rather than be answered with nothing. + @Test("A read that failed is not kept") + func failedReadIsNotKept() throws { + let url = try write([#"{"a":1}"#]) + defer { try? FileManager.default.removeItem(at: url) } + let cache = JSONFieldDetectionCache() + #expect(throws: CancellationError.self) { + _ = try cache.fields(at: url) { throw CancellationError() } + } + var reads = 0 + let fields = try cache.fields(at: url) { + reads += 1 + return try JSONImportParsing.detectFields(inLinesAt: url) + } + #expect(reads == 1) + #expect(fields.map(\.name) == ["a"]) + } +} diff --git a/TableProTests/Plugins/JSONImportFieldDetectionTests.swift b/TableProTests/Plugins/JSONImportFieldDetectionTests.swift index acaf9e23dc..01c7ef2d66 100644 --- a/TableProTests/Plugins/JSONImportFieldDetectionTests.swift +++ b/TableProTests/Plugins/JSONImportFieldDetectionTests.swift @@ -58,6 +58,34 @@ struct JSONImportFieldDetectionTests { #expect(fields.map(\.name) == ["id", "total"]) } + /// A JSON Lines file's fields do not depend on the table, and the import sheet asks again on + /// every table pick. The second ask is answered without opening the file, which a file that + /// can no longer be read proves: permissions are not part of what marks a file as changed. + @Test("Picking another table does not read a JSON Lines file again") + func anotherTableReusesTheFieldsOfAJSONLinesFile() throws { + let url = try write(Data("{\"id\":1,\"name\":\"Ada\"}\n".utf8), fileExtension: "ndjson") + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: url.path) + try? FileManager.default.removeItem(at: url) + } + let plugin = JSONImportPlugin() + #expect(try plugin.detectSourceFields(at: url, targetTable: "people").map(\.name) == ["id", "name"]) + + try FileManager.default.setAttributes([.posixPermissions: 0o000], ofItemAtPath: url.path) + + #expect(try plugin.detectSourceFields(at: url, targetTable: "authors").map(\.name) == ["id", "name"]) + } + + @Test("A table-keyed JSON file still gives each table its own fields") + func tableKeyedFileIsReadForEachTable() throws { + let json = #"{"users":[{"id":1,"email":"a@b.c"}],"orders":[{"id":1,"total":2.5}]}"# + let url = try write(Data(json.utf8), fileExtension: "json") + defer { try? FileManager.default.removeItem(at: url) } + let plugin = JSONImportPlugin() + #expect(try plugin.detectSourceFields(at: url, targetTable: "users").map(\.name) == ["email", "id"]) + #expect(try plugin.detectSourceFields(at: url, targetTable: "orders").map(\.name) == ["id", "total"]) + } + @Test("A field's type fits every value in the file, not only the first rows") func typeFitsEveryValue() throws { var lines = (1...300).map { #"{"code":\#($0)}"# } diff --git a/TableProTests/Plugins/JSONImportSkipTests.swift b/TableProTests/Plugins/JSONImportSkipTests.swift index b2ccbd11ca..dbfbdc623b 100644 --- a/TableProTests/Plugins/JSONImportSkipTests.swift +++ b/TableProTests/Plugins/JSONImportSkipTests.swift @@ -55,22 +55,19 @@ struct JSONImportSkipTests { private func runImport( _ lines: [String], errorHandling: ImportErrorHandling, - sink: CountingSink = CountingSink(), - progress: Progress = Progress() + sink: CountingSink = CountingSink() ) async throws -> Result { try await runImport( contents: Data(lines.joined(separator: "\n").utf8), errorHandling: errorHandling, - sink: sink, - progress: progress + sink: sink ) } private func runImport( contents: Data, errorHandling: ImportErrorHandling, - sink: CountingSink = CountingSink(), - progress: Progress = Progress() + sink: CountingSink = CountingSink() ) async throws -> Result { let url = try writeNDJSON(contents) defer { try? FileManager.default.removeItem(at: url) } @@ -88,7 +85,7 @@ struct JSONImportSkipTests { let result = try await plugin.performImport( source: FileSource(url: url), sink: sink, - progress: PluginImportProgress(progress: progress) + progress: PluginImportProgress(progress: Progress()) ) return .success(result) } catch { @@ -268,22 +265,4 @@ struct JSONImportSkipTests { #expect(finished) } - - /// The runner checks for a stop between batches, and a batch used to end only once it held 500 - /// rows. A long run of lines with no row in them was read to the end of the file first. - @Test("A stopped import stops within a batch of lines even when no line holds a row") - func stopReachesARunOfUnreadableLines() async throws { - let progress = Progress() - progress.cancel() - let outcome = try await runImport( - Array(repeating: "{ this is not json", count: 5_000), - errorHandling: .skipAndContinue, - progress: progress - ) - guard case .failure(let error) = outcome else { - Issue.record("A stopped import must not run to the end of the file: \(outcome)") - return - } - #expect(error is PluginImportCancellationError) - } } diff --git a/TableProTests/Plugins/JSONLineBatchesTests.swift b/TableProTests/Plugins/JSONLineBatchesTests.swift new file mode 100644 index 0000000000..227f164825 --- /dev/null +++ b/TableProTests/Plugins/JSONLineBatchesTests.swift @@ -0,0 +1,78 @@ +// +// JSONLineBatchesTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct JSONLineBatchesTests { + private func write(_ lines: [String]) throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("json-line-batches-\(UUID().uuidString).ndjson") + try Data(lines.map { $0 + "\n" }.joined().utf8).write(to: url) + return url + } + + private func batches(over url: URL, skipsUnreadableLines: Bool = true) throws -> JSONLineBatches { + JSONLineBatches( + lines: try JSONLineReader(url: url), + linesPerBatch: 500, + skipsUnreadableLines: skipsUnreadableLines, + maxRecordedErrors: 1_000 + ) + } + + /// The runner checks for a stop between batches. A batch used to end only once it held 500 + /// rows, so a run of lines holding none was read to the end of the reader's chunk, about + /// 55,000 such lines, before a stop was seen. + @Test("A batch ends after 500 lines even when none of them holds a row") + func batchEndsWithinARunOfUnreadableLines() throws { + let url = try write(Array(repeating: "{ this is not json", count: 5_000)) + defer { try? FileManager.default.removeItem(at: url) } + var lines = try batches(over: url) + defer { lines.close() } + + let first = try #require(try lines.next()) + + #expect(first.isEmpty) + #expect(lines.linesRead == 500) + #expect(lines.unreadableLineCount == 500) + } + + @Test("Batches cover every line once, each row under its own line number") + func batchesCoverEveryLine() throws { + let contents = (1...1_200).map { $0.isMultiple(of: 3) ? "{ this is not json" : #"{"n":\#($0)}"# } + let url = try write(contents) + defer { try? FileManager.default.removeItem(at: url) } + var lines = try batches(over: url) + defer { lines.close() } + + var batchCount = 0 + var entries: [RowImportRunner.Entry] = [] + while let batch = try lines.next() { + batchCount += 1 + entries.append(contentsOf: batch) + } + + #expect(batchCount == 3) + #expect(lines.linesRead == 1_200) + #expect(entries.count == 800) + #expect(lines.unreadableLineCount == 400) + #expect(entries.allSatisfy { $0.row["n"] == .text(String($0.line)) }) + } + + @Test("A stop mode throws at the first unreadable line") + func stopModeThrows() throws { + let url = try write([#"{"n":1}"#, "{ this is not json", #"{"n":3}"#]) + defer { try? FileManager.default.removeItem(at: url) } + var lines = try batches(over: url, skipsUnreadableLines: false) + defer { lines.close() } + + #expect(throws: (any Error).self) { + _ = try lines.next() + } + #expect(lines.linesRead == 2) + } +} diff --git a/TableProTests/Views/ImportPluginObservationTests.swift b/TableProTests/Views/ImportPluginObservationTests.swift new file mode 100644 index 0000000000..a29ff6e2b2 --- /dev/null +++ b/TableProTests/Views/ImportPluginObservationTests.swift @@ -0,0 +1,54 @@ +// +// ImportPluginObservationTests.swift +// TableProTests +// + +import Combine +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class DelimitedImportPlugin: ObservableObject, ImportFormatPlugin, @unchecked Sendable { + static let pluginName = "Delimited" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Detects fields split on a delimiter" + static let formatId = "delimited" + static let formatDisplayName = "Delimited" + static let acceptedFileExtensions = ["txt"] + static let iconName = "doc" + + @Published var delimiter = "," + + required init() {} + + var fieldDetectionSignature: String { delimiter } + + func performImport( + source: any PluginImportSource, + sink: any PluginImportDataSink, + progress: PluginImportProgress + ) async throws -> PluginImportResult { + PluginImportResult(executedStatements: 0, executionTime: 0) + } +} + +@MainActor +struct ImportPluginObservationTests { + /// The import sheet holds the plugin through `PluginManager` and its options view is the only + /// thing that observed the plugin, so picking a new delimiter never redrew the sheet and its + /// field list stayed on the fields read for the old one. + @Test("A changed plugin option redraws the import sheet") + func optionChangeIsRelayed() { + let plugin = DelimitedImportPlugin() + let observation = ImportPluginObservation(plugin: plugin) + var redraws = 0 + let subscription = observation.objectWillChange.sink { redraws += 1 } + defer { subscription.cancel() } + + plugin.delimiter = ";" + + #expect(redraws == 1) + #expect(plugin.fieldDetectionSignature == ";") + } +} diff --git a/project.yml b/project.yml index 33d21160b6..a858eb0c4f 100644 --- a/project.yml +++ b/project.yml @@ -533,11 +533,13 @@ targets: - Plugins/MarkdownExportPlugin/MarkdownExportModels.swift - Plugins/ParquetExportPlugin/ParquetExportModels.swift - Plugins/ParquetExportPlugin/ParquetTypeMapper.swift + - Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift - Plugins/JSONImportPlugin/JSONFieldSurvey.swift - Plugins/JSONImportPlugin/JSONImportOptions.swift - Plugins/JSONImportPlugin/JSONImportOptionsView.swift - Plugins/JSONImportPlugin/JSONImportParsing.swift - Plugins/JSONImportPlugin/JSONImportPlugin.swift + - Plugins/JSONImportPlugin/JSONLineBatches.swift - Plugins/JSONImportPlugin/JSONLineReader.swift - Plugins/MSSQLDriverPlugin/MSSQLObjectQueries.swift - Plugins/MSSQLDriverPlugin/MSSQLTypeQueries.swift From 84657311bad581a139f0837a1d42a13f9d053098 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 30 Sep 2026 18:58:35 +0700 Subject: [PATCH 3/3] fix(sync): use the model policy for unknown safe mode levels --- CHANGELOG.md | 1 + .../TableProSync/SyncRecordMapper.swift | 14 ++----------- .../SyncRecordMapperTests.swift | 21 +++++++++++++++++++ 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72ba7a92fe..1ec44e3762 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Unknown Safe Mode levels synced to iOS allowing writes without confirmation. - Save disabled for Kafka connections set to Verify Identity without a CA file. - Crash on `inf` or `nan`, text like `007` stored as a number, and blobs stored as text on remote libSQL. - External Clients level and Cloudflare, Cloud SQL, SOCKS and Tunnel Command settings reset by an iCloud sync. diff --git a/Packages/TableProCore/Sources/TableProSync/SyncRecordMapper.swift b/Packages/TableProCore/Sources/TableProSync/SyncRecordMapper.swift index 53a5a6fb18..cd1d9c03f8 100644 --- a/Packages/TableProCore/Sources/TableProSync/SyncRecordMapper.swift +++ b/Packages/TableProCore/Sources/TableProSync/SyncRecordMapper.swift @@ -187,22 +187,12 @@ public enum SyncRecordMapper { } private static func storedSafeModeLevel(in fields: SyncRecordFields) -> SafeModeLevel { - safeModeLevel( - fromWire: fields[.safeModeLevel] as? String, + SafeModeLevel( + wireValue: fields[.safeModeLevel] as? String, isReadOnly: (fields[.isReadOnly] as? Int64 ?? 0) != 0 ) } - private static func safeModeLevel(fromWire raw: String?, isReadOnly: Bool) -> SafeModeLevel { - guard let raw else { return isReadOnly ? .readOnly : .off } - if let level = SafeModeLevel(rawValue: raw) { return level } - switch raw { - case "silent": return .off - case "alert", "alertFull", "safeMode", "safeModeFull": return .confirmWrites - default: return isReadOnly ? .readOnly : .off - } - } - // MARK: - Update Existing CKRecord (preserves macOS-only fields) public static func updateRecord(_ record: CKRecord, with connection: DatabaseConnection) { diff --git a/Packages/TableProCore/Tests/TableProSyncTests/SyncRecordMapperTests.swift b/Packages/TableProCore/Tests/TableProSyncTests/SyncRecordMapperTests.swift index 0463cdcad4..2fa2afdfb4 100644 --- a/Packages/TableProCore/Tests/TableProSyncTests/SyncRecordMapperTests.swift +++ b/Packages/TableProCore/Tests/TableProSyncTests/SyncRecordMapperTests.swift @@ -86,4 +86,25 @@ struct SyncRecordMapperTests { let decoded = try #require(SyncRecordMapper.toConnection(record)) #expect(decoded.safeModeLevel == .confirmWrites) } + + @Test("An unrecognized wire value keeps the legacy read-only restriction") + func unknownWireValuePreservesReadOnly() throws { + let record = makeRawRecord(safeModeLevelRaw: "someFutureLevel", isReadOnly: true) + let decoded = try #require(SyncRecordMapper.toConnection(record)) + #expect(decoded.safeModeLevel == .readOnly) + } + + @Test("A rename preserves an unrecognized wire value and requires confirmation") + func renamePreservesUnknownWireValue() throws { + let record = makeRawRecord(safeModeLevelRaw: "someFutureLevel") + var connection = try #require(SyncRecordMapper.toConnection(record)) + connection.name = "Renamed" + + SyncRecordMapper.updateRecord(record, with: connection) + + #expect(record["safeModeLevel"] as? String == "someFutureLevel") + let decoded = try #require(SyncRecordMapper.toConnection(record)) + #expect(decoded.name == "Renamed") + #expect(decoded.safeModeLevel == .confirmWrites) + } }