diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ef8b9e5a..2500bf441 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Undo and Redo in a tab with unsaved edits replaying another tab's changes against the wrong rows. - Import sheet creating, emptying or filling tables in another database after a database switch in another window. - Import sheet discarding a new table's column edits when a CSV or Excel option changes. +- JSON import leaving out fields first seen after row 200, and typing columns from those rows alone. +- No fields found in a JSON Lines file with CRLF line endings. +- JSON Lines rows with U+2028, U+2029 or U+0085 in a string failing to import, and invalid UTF-8 imported as U+FFFD. +- JSON Lines import stalling while GitHub Copilot is enabled. - Table Transfer emptying a destination table, then failing, when two source columns map to one column. - MongoDB, Elasticsearch, Typesense and SurrealDB saving a long array or object shortened for display as the cut text. - MongoDB refusing text like `[DRAFT] Chapter one...` as a value shortened for display. diff --git a/Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift b/Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift new file mode 100644 index 000000000..ab5ac8053 --- /dev/null +++ b/Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift @@ -0,0 +1,52 @@ +// +// JSONFieldDetectionCache.swift +// JSONImportPlugin +// + +import Foundation +import os +import TableProPluginKit + +/// The fields of the last JSON Lines file read, kept against that file's identity. +/// +/// A JSON Lines file's fields do not depend on the table they are matched against, and reading +/// them means reading every line. The import sheet asks again on every destination table pick, so +/// each pick used to read the whole file again. +final class JSONFieldDetectionCache: Sendable { + /// A file counts as unchanged while its path, file number, size and modification date all + /// match. The identity is taken before the read, so an edit made during one leaves a stale + /// identity behind and the next request reads the file again. It describes the file a link + /// points to, because `attributesOfItem` describes the link itself and the read follows it. + private struct FileIdentity: Equatable, Sendable { + let path: String + let fileNumber: UInt64? + let size: UInt64? + let modificationDate: Date? + + init(of url: URL) throws { + let file = url.resolvingSymlinksInPath() + let attributes = try FileManager.default.attributesOfItem(atPath: file.path) + path = file.path + fileNumber = (attributes[.systemFileNumber] as? NSNumber)?.uint64Value + size = (attributes[.size] as? NSNumber)?.uint64Value + modificationDate = attributes[.modificationDate] as? Date + } + } + + private struct Entry: Sendable { + let file: FileIdentity + let fields: [PluginImportField] + } + + private let lastEntry = OSAllocatedUnfairLock(initialState: nil) + + func fields(at url: URL, detect: () throws -> [PluginImportField]) throws -> [PluginImportField] { + let file = try FileIdentity(of: url) + if let cached = lastEntry.withLock({ $0?.file == file ? $0?.fields : nil }) { + return cached + } + let fields = try detect() + lastEntry.withLock { $0 = Entry(file: file, fields: fields) } + return fields + } +} diff --git a/Plugins/JSONImportPlugin/JSONFieldSurvey.swift b/Plugins/JSONImportPlugin/JSONFieldSurvey.swift new file mode 100644 index 000000000..636afd94f --- /dev/null +++ b/Plugins/JSONImportPlugin/JSONFieldSurvey.swift @@ -0,0 +1,102 @@ +// +// JSONFieldSurvey.swift +// JSONImportPlugin +// + +import Foundation +import TableProPluginKit + +/// Every field any row of the file names, each with the first value it holds and a type that fits +/// every value it holds. It keeps one small record per field rather than the values themselves, +/// so it can read every row of a file of any size. +struct JSONFieldSurvey { + private var profiles: [String: JSONFieldProfile] = [:] + + mutating func add(_ row: NSDictionary) { + row.enumerateKeysAndObjects { key, value, _ in + guard let name = key as? String else { return } + profiles[name, default: JSONFieldProfile()].add(value) + } + } + + var fields: [PluginImportField] { + profiles.keys.sorted().map { name in + let profile = profiles[name] ?? JSONFieldProfile() + return PluginImportField( + name: name, + sampleValue: profile.sampleValue, + inferredType: profile.kinds.inferredType + ) + } + } +} + +struct JSONFieldProfile { + private(set) var sampleValue: String? + private(set) var kinds = JSONValueKinds() + + mutating func add(_ value: Any) { + let kind = JSONValueKind(of: value) + guard kind != .null else { return } + if sampleValue == nil { + sampleValue = JSONImportParsing.sampleString(value) + } + kinds.add(kind) + } +} + +struct JSONValueKinds { + private var sawValue = false + private var allNested = true + private var allBoolean = true + private var allInteger = true + private var allNumber = true + + mutating func add(_ kind: JSONValueKind) { + sawValue = true + if kind != .nested { allNested = false } + if kind != .boolean { allBoolean = false } + if kind != .integer { allInteger = false } + if kind != .integer, kind != .real { allNumber = false } + } + + var inferredType: PluginImportFieldType { + guard sawValue else { return .text } + if allNested { return .json } + if allBoolean { return .boolean } + if allInteger { return .integer } + if allNumber { return .real } + return .text + } +} + +/// Reads a parsed JSON value's kind from its Core Foundation type. A survey classifies every value +/// in the file, and a Swift `is` or `as?` cast from `Any` costs about ten times as much. +enum JSONValueKind: Equatable { + case null + case nested + case boolean + case integer + case real + case other + + init(of value: Any) { + let object = value as AnyObject + switch CFGetTypeID(object) { + case CFNullGetTypeID(): + self = .null + case CFArrayGetTypeID(), CFDictionaryGetTypeID(): + self = .nested + case CFBooleanGetTypeID(): + self = .boolean + case CFNumberGetTypeID(): + guard let number = object as? NSNumber else { + self = .other + return + } + self = CFNumberIsFloatType(number) ? .real : .integer + default: + self = .other + } + } +} diff --git a/Plugins/JSONImportPlugin/JSONImportParsing.swift b/Plugins/JSONImportPlugin/JSONImportParsing.swift index a27414eec..9389cba39 100644 --- a/Plugins/JSONImportPlugin/JSONImportParsing.swift +++ b/Plugins/JSONImportPlugin/JSONImportParsing.swift @@ -16,23 +16,32 @@ enum JSONImportParsing { ["jsonl", "ndjson"].contains(url.pathExtension.lowercased()) } - static func parseRow(fromLine line: String) throws -> [String: PluginCellValue] { - let object = try JSONSerialization.jsonObject(with: Data(line.utf8)) - guard let dict = object as? [String: Any] else { + static func parseRow(fromLine line: Data) throws -> [String: PluginCellValue]? { + try object(fromLine: line).map(convertRow) + } + + static func object(fromLine line: Data) throws -> NSDictionary? { + guard !isBlank(line) else { return nil } + let object = try JSONSerialization.jsonObject(with: line) + guard let dict = object as? NSDictionary else { throw PluginImportError.importFailed("Each line must be a JSON object") } - return convertRow(dict) + return dict + } + + private static func isBlank(_ line: Data) -> Bool { + line.allSatisfy { $0 == 0x20 || $0 == 0x09 || $0 == 0x0D } } - static func parseRows(at url: URL, targetTable: String?) throws -> [[String: Any]] { + static func parseRows(at url: URL, targetTable: String?) throws -> [NSDictionary] { let data = try Data(contentsOf: url) let object = try JSONSerialization.jsonObject(with: data) return try extractRows(from: object, targetTable: targetTable) } - static func extractRows(from object: Any, targetTable: String?) throws -> [[String: Any]] { + static func extractRows(from object: Any, targetTable: String?) throws -> [NSDictionary] { if let array = object as? [Any] { - return array.compactMap { $0 as? [String: Any] } + return array.compactMap { $0 as? NSDictionary } } guard let dict = object as? [String: Any] else { @@ -41,19 +50,19 @@ enum JSONImportParsing { let tables = dict.compactMapValues { value -> [Any]? in guard let array = value as? [Any] else { return nil } - return array.allSatisfy { $0 is [String: Any] } ? array : nil + return array.allSatisfy { $0 is NSDictionary } ? array : nil } let isTableWrapper = !tables.isEmpty && tables.count == dict.count guard isTableWrapper else { - return [dict] + return [dict as NSDictionary] } if let targetTable, let match = matchTable(in: tables, to: targetTable) { - return match.compactMap { $0 as? [String: Any] } + return match.compactMap { $0 as? NSDictionary } } if tables.count == 1, let only = tables.values.first { - return only.compactMap { $0 as? [String: Any] } + return only.compactMap { $0 as? NSDictionary } } throw PluginImportError.importFailed("The file contains multiple tables and none matches the target table") } @@ -68,8 +77,14 @@ enum JSONImportParsing { return suffix?.value } - static func convertRow(_ row: [String: Any]) -> [String: PluginCellValue] { - row.mapValues(cellValue(from:)) + static func convertRow(_ row: NSDictionary) -> [String: PluginCellValue] { + var converted: [String: PluginCellValue] = [:] + converted.reserveCapacity(row.count) + row.enumerateKeysAndObjects { key, value, _ in + guard let name = key as? String else { return } + converted[name] = cellValue(from: value) + } + return converted } static func cellValue(from json: Any) -> PluginCellValue { @@ -94,94 +109,37 @@ enum JSONImportParsing { // MARK: - Source introspection - static let sampleLength = 256 * 1_024 - - private static let longestUTF8Tail = 3 - - static func utf8Text(ofSample sample: Data) -> String { - for dropped in 0...min(longestUTF8Tail, sample.count) { - if let text = String(data: sample.dropLast(dropped), encoding: .utf8) { - return text - } + static func detectFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { + try Task.checkCancellation() + guard isLineDelimited(url) else { + return try detectFields(in: try parseRows(at: url, targetTable: targetTable)) } - return "" + return try detectFields(inLinesAt: url) } - static func sampleRawRows(at url: URL, targetTable: String?, limit: Int) throws -> [[String: Any]] { - if isLineDelimited(url) { - let handle = try FileHandle(forReadingFrom: url) - defer { try? handle.close() } - let text = utf8Text(ofSample: handle.readData(ofLength: sampleLength)) - var rows: [[String: Any]] = [] - for line in text.split(separator: "\n") where rows.count < limit { - let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { continue } - if let object = try? JSONSerialization.jsonObject(with: Data(trimmed.utf8)) as? [String: Any] { - rows.append(object) - } + static func detectFields(inLinesAt url: URL) throws -> [PluginImportField] { + var survey = JSONFieldSurvey() + var lines = try JSONLineReader(url: url) + defer { lines.close() } + while let line = try lines.next() { + autoreleasepool { + guard let row = try? object(fromLine: line) else { return } + survey.add(row) } - return rows } - let object = try JSONSerialization.jsonObject(with: Data(contentsOf: url)) - return Array(try extractRows(from: object, targetTable: targetTable).prefix(limit)) + return survey.fields } - static func detectFields(in rows: [[String: Any]]) -> [PluginImportField] { - var names: [String] = [] - var seen = Set() - var valuesByField: [String: [Any]] = [:] + static func detectFields(in rows: [NSDictionary]) throws -> [PluginImportField] { + var survey = JSONFieldSurvey() for row in rows { - for (key, value) in row { - if seen.insert(key).inserted { names.append(key) } - valuesByField[key, default: []].append(value) - } - } - return names.sorted().map { name in - let nonNull = (valuesByField[name] ?? []).filter { !($0 is NSNull) } - return PluginImportField( - name: name, - sampleValue: nonNull.first.map(sampleString), - inferredType: inferType(from: nonNull) - ) - } - } - - static func inferType(from values: [Any]) -> PluginImportFieldType { - guard !values.isEmpty else { return .text } - var allNested = true - var allBoolean = true - var allInteger = true - var allNumber = true - for value in values { - if value is [Any] || value is [String: Any] { - allBoolean = false - allInteger = false - allNumber = false - } else { - allNested = false - if let number = value as? NSNumber { - if CFGetTypeID(number) == CFBooleanGetTypeID() { - allInteger = false - allNumber = false - } else { - allBoolean = false - if CFNumberIsFloatType(number) { allInteger = false } - } - } else { - allBoolean = false - allInteger = false - allNumber = false - } - } + try Task.checkCancellation() + survey.add(row) } - if allNested { return .json } - if allBoolean { return .boolean } - if allInteger { return .integer } - if allNumber { return .real } - return .text + return survey.fields } - private static func sampleString(_ value: Any) -> String { + static func sampleString(_ value: Any) -> String { switch cellValue(from: value) { case .text(let string): return String(string.prefix(80)) case .bytes, .null: return "" diff --git a/Plugins/JSONImportPlugin/JSONImportPlugin.swift b/Plugins/JSONImportPlugin/JSONImportPlugin.swift index 19161b0b4..862385b76 100644 --- a/Plugins/JSONImportPlugin/JSONImportPlugin.swift +++ b/Plugins/JSONImportPlugin/JSONImportPlugin.swift @@ -42,6 +42,8 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug /// truncated list must not also under-report how much of the file was left out. private static let maxRecordedErrors = 1_000 + private let lineDelimitedFields = JSONFieldDetectionCache() + func performImport( source: any PluginImportSource, sink: any PluginImportDataSink, @@ -60,35 +62,20 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug var unreadableLineCount = 0 if JSONImportParsing.isLineDelimited(url) { progress.setEstimatedTotal(max(1, Int(source.fileSizeBytes() / 256))) - var lines = url.lines.makeAsyncIterator() - var lineNumber = 0 - let skipsErrors = settings.errorHandling == .skipAndContinue + var batches = JSONLineBatches( + lines: try JSONLineReader(url: url, checkCancellation: progress.checkCancellation), + linesPerBatch: Self.batchSize, + skipsUnreadableLines: settings.errorHandling == .skipAndContinue, + maxRecordedErrors: Self.maxRecordedErrors + ) + defer { batches.close() } outcome = try await RowImportRunner.run( configuration: configuration, sink: sink, progress: progress ) { - var batch: [RowImportRunner.Entry] = [] - while batch.count < Self.batchSize, let line = try await lines.next() { - lineNumber += 1 - let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { continue } - do { - let row = try JSONImportParsing.parseRow(fromLine: trimmed) - guard !row.isEmpty else { continue } - batch.append((lineNumber, row)) - } catch { - guard skipsErrors else { throw error } - unreadableLineCount += 1 - if unreadableLines.count < Self.maxRecordedErrors { - unreadableLines.append(.init( - statement: "row \(lineNumber)", - line: lineNumber, - errorMessage: error.localizedDescription - )) - } - } - } - return batch.isEmpty ? nil : batch + try batches.next() } + unreadableLines = batches.unreadableLines + unreadableLineCount = batches.unreadableLineCount } else { let rawRows = try JSONImportParsing.parseRows(at: url, targetTable: sink.targetTable) progress.setEstimatedTotal(rawRows.count) @@ -119,7 +106,11 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug // MARK: - Source introspection func detectSourceFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { - let rows = try JSONImportParsing.sampleRawRows(at: url, targetTable: targetTable, limit: 200) - return JSONImportParsing.detectFields(in: rows) + guard JSONImportParsing.isLineDelimited(url) else { + return try JSONImportParsing.detectFields(at: url, targetTable: targetTable) + } + return try lineDelimitedFields.fields(at: url) { + try JSONImportParsing.detectFields(inLinesAt: url) + } } } diff --git a/Plugins/JSONImportPlugin/JSONLineBatches.swift b/Plugins/JSONImportPlugin/JSONLineBatches.swift new file mode 100644 index 000000000..b377a77c3 --- /dev/null +++ b/Plugins/JSONImportPlugin/JSONLineBatches.swift @@ -0,0 +1,66 @@ +// +// JSONLineBatches.swift +// JSONImportPlugin +// + +import Foundation +import TableProPluginKit + +/// Hands the rows of a JSON Lines file to `RowImportRunner` one batch at a time, and keeps the +/// lines Skip and Continue passed over. +/// +/// A batch ends after a set number of lines, not rows. The runner checks for a stop between +/// batches, so a batch that waited for its rows read on through a run of lines holding none, up +/// to a whole chunk of the file, before a stop was seen. +struct JSONLineBatches { + private var lines: JSONLineReader + private let linesPerBatch: Int + private let skipsUnreadableLines: Bool + private let maxRecordedErrors: Int + + private(set) var unreadableLines: [PluginImportResult.ImportStatementError] = [] + + /// Every unreadable line, counted past the end of the capped `unreadableLines` list. + private(set) var unreadableLineCount = 0 + + var linesRead: Int { lines.lineNumber } + + init(lines: JSONLineReader, linesPerBatch: Int, skipsUnreadableLines: Bool, maxRecordedErrors: Int) { + self.lines = lines + self.linesPerBatch = max(1, linesPerBatch) + self.skipsUnreadableLines = skipsUnreadableLines + self.maxRecordedErrors = maxRecordedErrors + } + + mutating func next() throws -> [RowImportRunner.Entry]? { + var batch: [RowImportRunner.Entry] = [] + var linesInBatch = 0 + while linesInBatch < linesPerBatch, let line = try lines.next() { + linesInBatch += 1 + let lineNumber = lines.lineNumber + do { + let row = try autoreleasepool { try JSONImportParsing.parseRow(fromLine: line) } + guard let row, !row.isEmpty else { continue } + batch.append((lineNumber, row)) + } catch { + guard skipsUnreadableLines else { throw error } + recordUnreadableLine(lineNumber, error: error) + } + } + return linesInBatch == 0 ? nil : batch + } + + func close() { + lines.close() + } + + private mutating func recordUnreadableLine(_ lineNumber: Int, error: any Error) { + unreadableLineCount += 1 + guard unreadableLines.count < maxRecordedErrors else { return } + unreadableLines.append(.init( + statement: "row \(lineNumber)", + line: lineNumber, + errorMessage: error.localizedDescription + )) + } +} diff --git a/Plugins/JSONImportPlugin/JSONLineReader.swift b/Plugins/JSONImportPlugin/JSONLineReader.swift new file mode 100644 index 000000000..4195b4f3b --- /dev/null +++ b/Plugins/JSONImportPlugin/JSONLineReader.swift @@ -0,0 +1,88 @@ +// +// JSONLineReader.swift +// JSONImportPlugin +// + +import Foundation + +/// Reads a JSON Lines file one line at a time, holding one chunk and the line in progress. +/// +/// A line ends at a 0x0A byte and nowhere else, and its bytes go to the JSON parser undecoded. +/// Decoding a fixed-size prefix as text failed whenever the prefix ended inside a multi-byte +/// character. `URL.lines` also ends a line at U+2028, U+2029 and U+0085, which JSON allows +/// unescaped inside a string, and it reads through `FileHandle.AsyncBytes`, whose one +/// process-wide queue a reader waiting on a quiet pipe elsewhere in the app holds. +/// +/// A stop is checked before every chunk rather than between lines, because one line can run to +/// the end of a file that holds no newline at all. +struct JSONLineReader { + static let defaultChunkSize = 1 << 20 + + private let handle: FileHandle + private let chunkSize: Int + private let checkCancellation: () throws -> Void + private var buffer = Data() + private var lineStart = 0 + private var searchedUpTo = 0 + private var reachedEnd = false + + private(set) var lineNumber = 0 + + init( + url: URL, + chunkSize: Int = Self.defaultChunkSize, + checkCancellation: @escaping () throws -> Void = { try Task.checkCancellation() } + ) throws { + handle = try FileHandle(forReadingFrom: url) + self.chunkSize = max(1, chunkSize) + self.checkCancellation = checkCancellation + } + + mutating func next() throws -> Data? { + while true { + if let newline = firstNewline() { + let line = buffer[lineStart.. Int? { + buffer.withUnsafeBytes { raw -> Int? in + guard searchedUpTo < raw.count, let base = raw.baseAddress else { return nil } + guard let found = memchr(base + searchedUpTo, 0x0A, raw.count - searchedUpTo) else { return nil } + return base.distance(to: UnsafeRawPointer(found)) + } + } + + private mutating func refill() throws { + try checkCancellation() + if lineStart > 0 { + buffer.removeSubrange(0.. [PluginImportField] { + let detection = Task.detached { + try plugin.detectSourceFields(at: url, targetTable: targetTable) + } + return try await withTaskCancellationHandler { + try await detection.value + } onCancel: { + detection.cancel() + } + } +} diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 0da96c642..d4f3b6293 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -122610,6 +122610,9 @@ }, "Reading Tables" : { + }, + "Reading the file…" : { + }, "Reading the source…" : { diff --git a/TablePro/Views/Import/RowImportSheet.swift b/TablePro/Views/Import/RowImportSheet.swift index 8c5722e5f..a2ea4b97e 100644 --- a/TablePro/Views/Import/RowImportSheet.swift +++ b/TablePro/Views/Import/RowImportSheet.swift @@ -365,13 +365,13 @@ struct RowImportSheet: View { if selectedTargetTable == nil { placeholder("Choose a destination table to map fields.") } else if mapping.rows.isEmpty { - placeholder("No fields found in the file.") + placeholder(currentReadHasLanded ? "No fields found in the file." : readingPlaceholder) } else { mappingTable } case .newTable: if newTable.columns.isEmpty { - placeholder("No columns found in the file.") + placeholder(currentReadHasLanded ? "No columns found in the file." : readingPlaceholder) } else { newColumnsTable } @@ -379,6 +379,12 @@ struct RowImportSheet: View { } } + /// A read takes seconds on a large file, and until it lands the list is empty, which the other + /// placeholders would report as a file with nothing in it. + private var readingPlaceholder: String { + String(localized: "Reading the file…") + } + /// A file the plugin could not read is a failure, not an empty result. Showing the parser's /// message as grey placeholder text left the sheet with nothing to press but Cancel. private func unreadableFile(reason: String) -> some View { @@ -698,6 +704,11 @@ struct RowImportSheet: View { /// table and database, that finished without an error. private var currentReadIsReady: Bool { guard !isLoadingContext, loadError == nil else { return false } + return currentReadHasLanded + } + + /// The rows on screen answer the read for the current destination, table, database and options. + private var currentReadHasLanded: Bool { switch destination { case .existingTable: return mapping.loadedRead == AnyHashable(sourceRead) @@ -764,19 +775,6 @@ struct RowImportSheet: View { newTableName = suggestion } - /// `detectSourceFields` is synchronous and reads the file: the XLSX plugin materialises the - /// whole workbook, the CSV one reads a megabyte. Every state write stays on the main actor, - /// only the parse leaves it. - nonisolated private static func detectFields( - plugin: any ImportFormatPlugin, - at url: URL, - targetTable: String? - ) async throws -> [PluginImportField] { - try await Task.detached { - try plugin.detectSourceFields(at: url, targetTable: targetTable) - }.value - } - private var sourceRead: SourceRead { let isExisting = destination == .existingTable return SourceRead( @@ -826,7 +824,7 @@ struct RowImportSheet: View { isLoadingContext = true loadError = nil do { - let fields = try await Self.detectFields(plugin: plugin, at: fileURL, targetTable: nil) + let fields = try await ImportFieldDetection.detectFields(plugin: plugin, at: fileURL, targetTable: nil) guard !Task.isCancelled else { return } let serverVersion = DatabaseManager.shared.driver(for: connection.id)?.serverVersion newTable.load(fields: fields) { inferredType in @@ -854,7 +852,7 @@ struct RowImportSheet: View { try await driver.fetchColumns(table: table) }.map(\.name) guard !Task.isCancelled else { return } - let fields = try await Self.detectFields(plugin: plugin, at: fileURL, targetTable: table) + let fields = try await ImportFieldDetection.detectFields(plugin: plugin, at: fileURL, targetTable: table) guard !Task.isCancelled else { return } mapping.load(fields: fields, columns: columns, for: TableScope(table: table, in: scope), read: request) } catch { diff --git a/TableProTests/Core/Plugins/ImportFieldDetectionTests.swift b/TableProTests/Core/Plugins/ImportFieldDetectionTests.swift new file mode 100644 index 000000000..202cd509c --- /dev/null +++ b/TableProTests/Core/Plugins/ImportFieldDetectionTests.swift @@ -0,0 +1,74 @@ +// +// ImportFieldDetectionTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class BlockingDetectionPlugin: ImportFormatPlugin, @unchecked Sendable { + static let pluginName = "Blocking Detection" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Detects fields until it is cancelled" + static let formatId = "blocking-detection" + static let formatDisplayName = "Blocking" + static let acceptedFileExtensions = ["blocking"] + static let iconName = "doc" + + private let lock = NSLock() + private var started = false + private var cancelled = false + + required init() {} + + var hasStarted: Bool { lock.withLock { started } } + var sawCancellation: Bool { lock.withLock { cancelled } } + + func performImport( + source: any PluginImportSource, + sink: any PluginImportDataSink, + progress: PluginImportProgress + ) async throws -> PluginImportResult { + PluginImportResult(executedStatements: 0, executionTime: 0) + } + + func detectSourceFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { + lock.withLock { started = true } + let deadline = Date().addingTimeInterval(10) + while !Task.isCancelled, Date() < deadline { + usleep(1_000) + } + guard Task.isCancelled else { return [] } + lock.withLock { cancelled = true } + throw CancellationError() + } +} + +struct ImportFieldDetectionTests { + /// The import sheet reads the file from a detached task. Nothing cancelled that task, so closing + /// the sheet left a whole-file read running to the end. + @Test("Cancelling the caller cancels a detection already reading the file") + func cancellingTheCallerStopsTheRead() async throws { + let plugin = BlockingDetectionPlugin() + let caller = Task { + try await ImportFieldDetection.detectFields( + plugin: plugin, + at: URL(fileURLWithPath: "/dev/null"), + targetTable: nil + ) + } + for _ in 0..<500 where !plugin.hasStarted { + try await Task.sleep(nanoseconds: 10_000_000) + } + try #require(plugin.hasStarted) + + caller.cancel() + + await #expect(throws: CancellationError.self) { + _ = try await caller.value + } + #expect(plugin.sawCancellation) + } +} diff --git a/TableProTests/Plugins/JSONFieldDetectionCacheTests.swift b/TableProTests/Plugins/JSONFieldDetectionCacheTests.swift new file mode 100644 index 000000000..80857887c --- /dev/null +++ b/TableProTests/Plugins/JSONFieldDetectionCacheTests.swift @@ -0,0 +1,107 @@ +// +// JSONFieldDetectionCacheTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct JSONFieldDetectionCacheTests { + private func write(_ lines: [String]) throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("json-field-cache-\(UUID().uuidString).ndjson") + try Data(lines.map { $0 + "\n" }.joined().utf8).write(to: url) + return url + } + + private func append(_ line: String, to url: URL) throws { + let handle = try FileHandle(forWritingTo: url) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: Data((line + "\n").utf8)) + } + + @Test("An unchanged file is read once however often its fields are asked for") + func unchangedFileIsReadOnce() throws { + let url = try write([#"{"a":1}"#]) + defer { try? FileManager.default.removeItem(at: url) } + let cache = JSONFieldDetectionCache() + var reads = 0 + for _ in 0..<3 { + let fields = try cache.fields(at: url) { + reads += 1 + return try JSONImportParsing.detectFields(inLinesAt: url) + } + #expect(fields.map(\.name) == ["a"]) + } + #expect(reads == 1) + } + + @Test("A file changed since its last read is read again") + func changedFileIsReadAgain() throws { + let url = try write([#"{"a":1}"#]) + defer { try? FileManager.default.removeItem(at: url) } + let cache = JSONFieldDetectionCache() + _ = try cache.fields(at: url) { try JSONImportParsing.detectFields(inLinesAt: url) } + + try append(#"{"b":2}"#, to: url) + + let fields = try cache.fields(at: url) { try JSONImportParsing.detectFields(inLinesAt: url) } + #expect(fields.map(\.name) == ["a", "b"]) + } + + /// The read follows a symbolic link and file attributes do not, so an identity taken from the + /// link itself stayed the same while the file it points to changed. + @Test("A file reached through a symbolic link is read again once the file changes") + func changedFileBehindALinkIsReadAgain() throws { + let target = try write([#"{"a":1}"#]) + let link = target.deletingLastPathComponent() + .appendingPathComponent("json-field-cache-link-\(UUID().uuidString).ndjson") + try FileManager.default.createSymbolicLink(at: link, withDestinationURL: target) + defer { + try? FileManager.default.removeItem(at: link) + try? FileManager.default.removeItem(at: target) + } + let cache = JSONFieldDetectionCache() + _ = try cache.fields(at: link) { try JSONImportParsing.detectFields(inLinesAt: link) } + + try append(#"{"b":2}"#, to: target) + + let fields = try cache.fields(at: link) { try JSONImportParsing.detectFields(inLinesAt: link) } + #expect(fields.map(\.name) == ["a", "b"]) + } + + @Test("Another file is not answered with the fields of the last one") + func otherFileIsReadItself() throws { + let first = try write([#"{"a":1}"#]) + let second = try write([#"{"b":1}"#]) + defer { + try? FileManager.default.removeItem(at: first) + try? FileManager.default.removeItem(at: second) + } + let cache = JSONFieldDetectionCache() + _ = try cache.fields(at: first) { try JSONImportParsing.detectFields(inLinesAt: first) } + let fields = try cache.fields(at: second) { try JSONImportParsing.detectFields(inLinesAt: second) } + #expect(fields.map(\.name) == ["b"]) + } + + /// Closing the import sheet cancels the read. A cancelled read has no fields to keep, and the + /// next request has to read the file rather than be answered with nothing. + @Test("A read that failed is not kept") + func failedReadIsNotKept() throws { + let url = try write([#"{"a":1}"#]) + defer { try? FileManager.default.removeItem(at: url) } + let cache = JSONFieldDetectionCache() + #expect(throws: CancellationError.self) { + _ = try cache.fields(at: url) { throw CancellationError() } + } + var reads = 0 + let fields = try cache.fields(at: url) { + reads += 1 + return try JSONImportParsing.detectFields(inLinesAt: url) + } + #expect(reads == 1) + #expect(fields.map(\.name) == ["a"]) + } +} diff --git a/TableProTests/Plugins/JSONImportFieldDetectionTests.swift b/TableProTests/Plugins/JSONImportFieldDetectionTests.swift new file mode 100644 index 000000000..01c7ef2d6 --- /dev/null +++ b/TableProTests/Plugins/JSONImportFieldDetectionTests.swift @@ -0,0 +1,170 @@ +// +// JSONImportFieldDetectionTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct JSONImportFieldDetectionTests { + private func write(_ bytes: Data, fileExtension: String) throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("json-import-detection-\(UUID().uuidString).\(fileExtension)") + try bytes.write(to: url) + return url + } + + private func detect(_ bytes: Data, fileExtension: String, targetTable: String? = nil) throws -> [PluginImportField] { + let url = try write(bytes, fileExtension: fileExtension) + defer { try? FileManager.default.removeItem(at: url) } + return try JSONImportParsing.detectFields(at: url, targetTable: targetTable) + } + + private func detectLines(_ lines: [String]) throws -> [PluginImportField] { + try detect(Data(lines.map { $0 + "\n" }.joined().utf8), fileExtension: "ndjson") + } + + private func field(_ name: String, in fields: [PluginImportField]) throws -> PluginImportField { + try #require(fields.first { $0.name == name }) + } + + /// TablePro's own JSON export leaves a null key out of a row, so a column can first appear + /// deep into the file. Detection used to read 200 rows and never offer it. + @Test("A key first seen after row 200 of a JSON Lines file is detected") + func lateKeyInJSONLines() throws { + var lines = (1...200).map { #"{"id":\#($0)}"# } + lines.append(#"{"id":201,"note":"late","score":1.5}"#) + let fields = try detectLines(lines) + #expect(fields.map(\.name) == ["id", "note", "score"]) + #expect(try field("note", in: fields).sampleValue == "late") + #expect(try field("note", in: fields).inferredType == .text) + #expect(try field("score", in: fields).inferredType == .real) + } + + @Test("A key first seen after element 200 of a JSON array is detected") + func lateKeyInJSONArray() throws { + var elements = (1...250).map { #"{"id":\#($0)}"# } + elements[229] = #"{"id":230,"deleted_at":"2026-01-02"}"# + let fields = try detect(Data("[\(elements.joined(separator: ","))]".utf8), fileExtension: "json") + #expect(fields.map(\.name) == ["deleted_at", "id"]) + #expect(try field("deleted_at", in: fields).sampleValue == "2026-01-02") + } + + @Test("A table-keyed file reports the fields of the target table") + func tableKeyedFileUsesTheTargetTable() throws { + let json = #"{"users":[{"id":1}],"orders":[{"id":1},{"id":2,"total":2.5}]}"# + let fields = try detect(Data(json.utf8), fileExtension: "json", targetTable: "orders") + #expect(fields.map(\.name) == ["id", "total"]) + } + + /// A JSON Lines file's fields do not depend on the table, and the import sheet asks again on + /// every table pick. The second ask is answered without opening the file, which a file that + /// can no longer be read proves: permissions are not part of what marks a file as changed. + @Test("Picking another table does not read a JSON Lines file again") + func anotherTableReusesTheFieldsOfAJSONLinesFile() throws { + let url = try write(Data("{\"id\":1,\"name\":\"Ada\"}\n".utf8), fileExtension: "ndjson") + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: url.path) + try? FileManager.default.removeItem(at: url) + } + let plugin = JSONImportPlugin() + #expect(try plugin.detectSourceFields(at: url, targetTable: "people").map(\.name) == ["id", "name"]) + + try FileManager.default.setAttributes([.posixPermissions: 0o000], ofItemAtPath: url.path) + + #expect(try plugin.detectSourceFields(at: url, targetTable: "authors").map(\.name) == ["id", "name"]) + } + + @Test("A table-keyed JSON file still gives each table its own fields") + func tableKeyedFileIsReadForEachTable() throws { + let json = #"{"users":[{"id":1,"email":"a@b.c"}],"orders":[{"id":1,"total":2.5}]}"# + let url = try write(Data(json.utf8), fileExtension: "json") + defer { try? FileManager.default.removeItem(at: url) } + let plugin = JSONImportPlugin() + #expect(try plugin.detectSourceFields(at: url, targetTable: "users").map(\.name) == ["email", "id"]) + #expect(try plugin.detectSourceFields(at: url, targetTable: "orders").map(\.name) == ["id", "total"]) + } + + @Test("A field's type fits every value in the file, not only the first rows") + func typeFitsEveryValue() throws { + var lines = (1...300).map { #"{"code":\#($0)}"# } + lines.append(#"{"code":"A-301"}"#) + #expect(try field("code", in: detectLines(lines)).inferredType == .text) + } + + @Test("A key first seen past the first megabyte is detected") + func lateKeyPastTheFirstChunk() throws { + let padding = String(repeating: "x", count: 1_000) + var lines = Array(repeating: #"{"pad":"\#(padding)"}"#, count: 1_200) + lines.append(#"{"pad":"y","city":"Hà Nội"}"#) + let fields = try detectLines(lines) + #expect(try field("city", in: fields).sampleValue == "Hà Nội") + } + + /// Detection decoded the first 262,144 bytes as UTF-8 text, which fails outright when that + /// byte falls inside a character, so the whole file read as having no fields. + @Test("Text whose characters straddle the old prefix and the read chunk is detected") + func multiByteCharacterAtTheOldPrefixEnd() throws { + var bytes = Data() + let namePrefixLength = #"{"name":""#.utf8.count + func appendLine(_ line: String) { + bytes.append(Data((line + "\n").utf8)) + } + func appendPadding(untilNextLineStartsAt start: Int) { + let fixedLength = #"{"pad":""}"#.utf8.count + 1 + appendLine(#"{"pad":""# + String(repeating: "x", count: start - bytes.count - fixedLength) + #""}"#) + } + appendPadding(untilNextLineStartsAt: 262_144 - 1 - namePrefixLength) + appendLine(#"{"name":"東京"}"#) + appendPadding(untilNextLineStartsAt: JSONLineReader.defaultChunkSize - 1 - namePrefixLength) + appendLine(#"{"name":"東京","city":"Hà Nội"}"#) + try #require(bytes[262_144] & 0xC0 == 0x80) + try #require(bytes[JSONLineReader.defaultChunkSize] & 0xC0 == 0x80) + + let fields = try detect(bytes, fileExtension: "ndjson") + #expect(fields.map(\.name) == ["city", "name", "pad"]) + #expect(try field("name", in: fields).sampleValue == "東京") + #expect(try field("city", in: fields).sampleValue == "Hà Nội") + } + + @Test("A JSON Lines file with CRLF line endings is detected") + func crlfLineEndings() throws { + let fields = try detect(Data("{\"a\":1}\r\n{\"a\":2,\"b\":\"x\"}\r\n".utf8), fileExtension: "jsonl") + #expect(fields.map(\.name) == ["a", "b"]) + #expect(try field("a", in: fields).inferredType == .integer) + } + + @Test("A string holding U+2028 stays one row") + func lineSeparatorInsideAString() throws { + let fields = try detectLines(["{\"note\":\"a\u{2028}b\"}"]) + #expect(try field("note", in: fields).sampleValue == "a\u{2028}b") + } + + @Test("Unreadable and non-object lines are passed over") + func unreadableLinesArePassedOver() throws { + let fields = try detectLines([#"{"a":1}"#, "{ this is not json", "[1, 2]", "", #"{"b":2}"#]) + #expect(fields.map(\.name) == ["a", "b"]) + } + + @Test("A field that is only ever null is detected as text with no sample") + func nullOnlyField() throws { + let fields = try detectLines([#"{"a":null}"#, #"{"a":null}"#]) + #expect(try field("a", in: fields).inferredType == .text) + #expect(try field("a", in: fields).sampleValue == nil) + } + + @Test("Detection stops once its task is cancelled", arguments: ["ndjson", "json"]) + func cancelledDetectionStops(fileExtension: String) async throws { + let contents = fileExtension == "json" ? #"[{"a":1}]"# : "{\"a\":1}\n" + let url = try write(Data(contents.utf8), fileExtension: fileExtension) + defer { try? FileManager.default.removeItem(at: url) } + let detection = Task.detached { + withUnsafeCurrentTask { $0?.cancel() } + return try JSONImportParsing.detectFields(at: url, targetTable: nil) + } + await #expect(throws: CancellationError.self) { + _ = try await detection.value + } + } +} diff --git a/TableProTests/Plugins/JSONImportPluginTests.swift b/TableProTests/Plugins/JSONImportPluginTests.swift index d261301f3..284cbfc90 100644 --- a/TableProTests/Plugins/JSONImportPluginTests.swift +++ b/TableProTests/Plugins/JSONImportPluginTests.swift @@ -8,9 +8,9 @@ import TableProPluginKit import Testing struct JSONImportPluginTests { - private func object(_ json: String) throws -> [String: Any] { + private func object(_ json: String) throws -> NSDictionary { let parsed = try JSONSerialization.jsonObject(with: Data(json.utf8)) - return try #require(parsed as? [String: Any]) + return try #require(parsed as? NSDictionary) } private func anyValue(_ json: String) throws -> Any { @@ -110,7 +110,7 @@ struct JSONImportPluginTests { @Test("A JSON object line parses to a row") func testNdjsonLine() throws { - let row = try JSONImportParsing.parseRow(fromLine: #"{"id":1,"name":"x"}"#) + let row = try #require(try JSONImportParsing.parseRow(fromLine: Data(#"{"id":1,"name":"x"}"#.utf8))) #expect(row["id"] == .text("1")) #expect(row["name"] == .text("x")) } @@ -118,7 +118,27 @@ struct JSONImportPluginTests { @Test("A non-object line throws") func testNdjsonNonObjectThrows() { #expect(throws: PluginImportError.self) { - _ = try JSONImportParsing.parseRow(fromLine: "[1, 2, 3]") + _ = try JSONImportParsing.parseRow(fromLine: Data("[1, 2, 3]".utf8)) + } + } + + @Test("A line of JSON whitespace is blank, not an error") + func testNdjsonBlankLine() throws { + #expect(try JSONImportParsing.parseRow(fromLine: Data()) == nil) + #expect(try JSONImportParsing.parseRow(fromLine: Data(" \t\r".utf8)) == nil) + } + + @Test("A line ending in a carriage return parses") + func testNdjsonCarriageReturnLine() throws { + let row = try #require(try JSONImportParsing.parseRow(fromLine: Data("{\"id\":1}\r".utf8))) + #expect(row["id"] == .text("1")) + } + + @Test("A line whose bytes are not UTF-8 throws rather than importing replacement characters") + func testNdjsonInvalidUTF8Throws() { + let line = Data(#"{"name":""#.utf8) + Data([0xFF]) + Data(#""}"#.utf8) + #expect(throws: (any Error).self) { + _ = try JSONImportParsing.parseRow(fromLine: line) } } @@ -142,41 +162,54 @@ struct JSONImportPluginTests { try #require(try JSONSerialization.jsonObject(with: Data(json.utf8)) as? [Any]) } + private func inferredType(_ json: String) throws -> PluginImportFieldType { + var kinds = JSONValueKinds() + for value in try array(json) { + kinds.add(JSONValueKind(of: value)) + } + return kinds.inferredType + } + @Test("Inference: all integers") func testInferInteger() throws { - #expect(JSONImportParsing.inferType(from: try array("[1, 2, 3]")) == .integer) + #expect(try inferredType("[1, 2, 3]") == .integer) } @Test("Inference: any decimal makes the field real") func testInferReal() throws { - #expect(JSONImportParsing.inferType(from: try array("[1, 2.5, 3]")) == .real) + #expect(try inferredType("[1, 2.5, 3]") == .real) } @Test("Inference: all booleans") func testInferBoolean() throws { - #expect(JSONImportParsing.inferType(from: try array("[true, false]")) == .boolean) + #expect(try inferredType("[true, false]") == .boolean) } @Test("Inference: all-nested values are json") func testInferJSON() throws { - #expect(JSONImportParsing.inferType(from: try array(#"[{"a":1}, [1,2]]"#)) == .json) + #expect(try inferredType(#"[{"a":1}, [1,2]]"#) == .json) } @Test("Inference: mixed types fall back to text") func testInferText() throws { - #expect(JSONImportParsing.inferType(from: try array(#"["a", 1]"#)) == .text) + #expect(try inferredType(#"["a", 1]"#) == .text) + } + + @Test("Inference: a value after the type settles on text keeps it text") + func testInferTextIsFinal() throws { + #expect(try inferredType(#"["a", 1, true, {"k":1}]"#) == .text) } @Test("Inference: empty values are text") - func testInferEmpty() { - #expect(JSONImportParsing.inferType(from: []) == .text) + func testInferEmpty() throws { + #expect(try inferredType("[]") == .text) } @Test("detectFields reports sorted fields with inferred types and a sample") func testDetectFields() throws { let raw = #"[{"id":1,"name":"a","active":true},{"id":2,"name":"b","active":false}]"# - let rows = try #require(try JSONSerialization.jsonObject(with: Data(raw.utf8)) as? [[String: Any]]) - let fields = JSONImportParsing.detectFields(in: rows) + let rows = try #require(try JSONSerialization.jsonObject(with: Data(raw.utf8)) as? [NSDictionary]) + let fields = try JSONImportParsing.detectFields(in: rows) #expect(fields.map(\.name) == ["active", "id", "name"]) #expect(fields.first { $0.name == "id" }?.inferredType == .integer) #expect(fields.first { $0.name == "active" }?.inferredType == .boolean) @@ -184,17 +217,18 @@ struct JSONImportPluginTests { #expect(fields.first { $0.name == "id" }?.sampleValue == "1") } - // MARK: - JSON Lines sample + // MARK: - JSON Lines detection - @Test("A JSON Lines sample cut inside a Japanese character still lists its fields") - func testSampleCutInsideAMultiByteCharacter() throws { + @Test("A JSON Lines file whose 256 KB mark falls inside a Japanese character still lists its fields") + func testDetectionAcrossACutMultiByteCharacter() throws { let line = Data("{\"名前\":\"山田太郎\",\"住所\":\"東京都港区\"}\n".utf8) + let mark = 256 * 1_024 var body = Data() - while body.count <= JSONImportParsing.sampleLength { + while body.count <= mark { body.append(line) } var file = body - while String(data: file.prefix(JSONImportParsing.sampleLength), encoding: .utf8) != nil { + while String(data: file.prefix(mark), encoding: .utf8) != nil { file.insert(0x0A, at: 0) } @@ -202,14 +236,8 @@ struct JSONImportPluginTests { .appendingPathComponent("JSONImportPluginTests-\(UUID().uuidString).jsonl") try file.write(to: url) defer { try? FileManager.default.removeItem(at: url) } - let rows = try JSONImportParsing.sampleRawRows(at: url, targetTable: nil, limit: 5) - #expect(rows.count == 5) - #expect(rows.first?["名前"] as? String == "山田太郎") - } - - @Test("A sample of whole characters decodes unchanged") - func testSampleOfWholeCharacters() { - #expect(JSONImportParsing.utf8Text(ofSample: Data("{\"a\":\"日本\"}".utf8)) == "{\"a\":\"日本\"}") - #expect(JSONImportParsing.utf8Text(ofSample: Data([0x7B, 0xE6, 0x97])) == "{") + let fields = try JSONImportParsing.detectFields(at: url, targetTable: nil) + #expect(Set(fields.map(\.name)) == ["名前", "住所"]) + #expect(fields.first { $0.name == "名前" }?.sampleValue == "山田太郎") } } diff --git a/TableProTests/Plugins/JSONImportSkipTests.swift b/TableProTests/Plugins/JSONImportSkipTests.swift index b7d512746..dbfbdc623 100644 --- a/TableProTests/Plugins/JSONImportSkipTests.swift +++ b/TableProTests/Plugins/JSONImportSkipTests.swift @@ -11,11 +11,11 @@ private final class CountingSink: PluginImportDataSink, @unchecked Sendable { let databaseTypeId = "mock" let targetTable: String? = "people" - private(set) var insertedRows = 0 + private(set) var rows: [[String: PluginCellValue]] = [] func execute(statement: String) async throws {} - func insertRow(_ values: [String: PluginCellValue]) async throws { insertedRows += 1 } - func insertRows(_ rows: [[String: PluginCellValue]]) async throws { insertedRows += rows.count } + func insertRow(_ values: [String: PluginCellValue]) async throws { rows.append(values) } + func insertRows(_ rows: [[String: PluginCellValue]]) async throws { self.rows.append(contentsOf: rows) } func deleteAllRowsFromTargetTable() async throws {} func beginTransaction() async throws {} func commitTransaction() async throws {} @@ -45,18 +45,31 @@ private final class FileSource: PluginImportSource, @unchecked Sendable { /// instances in flight at once read each other's error-handling mode. @Suite("JSON import skips unreadable lines", .serialized) struct JSONImportSkipTests { - private func writeNDJSON(_ lines: [String]) throws -> URL { + private func writeNDJSON(_ contents: Data) throws -> URL { let url = FileManager.default.temporaryDirectory .appendingPathComponent("json-import-\(UUID().uuidString).ndjson") - try lines.joined(separator: "\n").write(to: url, atomically: true, encoding: .utf8) + try contents.write(to: url) return url } private func runImport( _ lines: [String], - errorHandling: ImportErrorHandling + errorHandling: ImportErrorHandling, + sink: CountingSink = CountingSink() ) async throws -> Result { - let url = try writeNDJSON(lines) + try await runImport( + contents: Data(lines.joined(separator: "\n").utf8), + errorHandling: errorHandling, + sink: sink + ) + } + + private func runImport( + contents: Data, + errorHandling: ImportErrorHandling, + sink: CountingSink = CountingSink() + ) async throws -> Result { + let url = try writeNDJSON(contents) defer { try? FileManager.default.removeItem(at: url) } /// `settings` persists through plugin storage, so a test that writes it changes the @@ -71,7 +84,7 @@ struct JSONImportSkipTests { do { let result = try await plugin.performImport( source: FileSource(url: url), - sink: CountingSink(), + sink: sink, progress: PluginImportProgress(progress: Progress()) ) return .success(result) @@ -166,4 +179,90 @@ struct JSONImportSkipTests { #expect(result.skippedStatements == 0) #expect(result.errors.isEmpty) } + + /// JSON allows U+2028, U+2029 and U+0085 unescaped inside a string. Reading the file with + /// `URL.lines` ended a line at each of them and failed both halves. + @Test("A string holding a Unicode line separator imports as one row") + func unicodeSeparatorInsideAStringImportsWhole() async throws { + let sink = CountingSink() + let outcome = try await runImport( + ["{\"note\":\"a\u{2028}b\u{2029}c\u{0085}d\"}", #"{"note": "e"}"#], + errorHandling: .skipAndContinue, + sink: sink + ) + guard case .success(let result) = outcome else { + Issue.record("A valid file must import: \(outcome)") + return + } + #expect(result.executedStatements == 2) + #expect(result.skippedStatements == 0) + let first = try #require(sink.rows.first) + #expect(first["note"] == .text("a\u{2028}b\u{2029}c\u{0085}d")) + } + + @Test("Lines ending in CRLF import every row") + func crlfLinesImport() async throws { + let outcome = try await runImport( + contents: Data("{\"a\":1}\r\n\r\n{\"a\":2}\r\n".utf8), + errorHandling: .stopAndRollback + ) + guard case .success(let result) = outcome else { + Issue.record("A CRLF file must import: \(outcome)") + return + } + #expect(result.executedStatements == 2) + } + + /// The same bytes in a `.json` file fail the whole parse. A JSON Lines file used to have the + /// bad byte swapped for U+FFFD and imported as if nothing were wrong. + @Test("A line that is not UTF-8 is reported rather than imported with replacement characters") + func invalidUTF8LineIsReported() async throws { + var contents = Data("{\"name\": \"Ada\"}\n{\"name\": \"".utf8) + contents.append(0xFF) + contents.append(Data("\"}\n{\"name\": \"Grace\"}".utf8)) + let sink = CountingSink() + let outcome = try await runImport(contents: contents, errorHandling: .skipAndContinue, sink: sink) + guard case .success(let result) = outcome else { + Issue.record("Skip and Continue must not abort the import: \(outcome)") + return + } + #expect(result.executedStatements == 2) + #expect(result.skippedStatements == 1) + #expect(result.errors.contains { $0.line == 2 }) + #expect(sink.rows.compactMap { $0["name"] } == [.text("Ada"), .text("Grace")]) + } + + /// `URL.lines` reads through `FileHandle.AsyncBytes`, which Foundation serves from one queue for + /// the whole process. A reader parked on a quiet pipe, as the Copilot language server's is, + /// held that queue and the import waited behind it. + @Test("An import finishes while another reader in the process waits on a quiet pipe") + func importIgnoresABlockedAsyncBytesReader() async throws { + let pipe = Pipe() + let blocker = Task { + for try await _ in pipe.fileHandleForReading.bytes {} + } + try await Task.sleep(nanoseconds: 100_000_000) + + let finished = await withTaskGroup(of: Bool.self) { group in + group.addTask { + let outcome = try? await self.runImport( + [#"{"name": "Ada"}"#, #"{"name": "Grace"}"#], + errorHandling: .stopAndRollback + ) + guard case .success(let result) = outcome else { return false } + return result.executedStatements == 2 + } + group.addTask { + try? await Task.sleep(nanoseconds: 10_000_000_000) + return false + } + let first = await group.next() ?? false + try? pipe.fileHandleForWriting.close() + group.cancelAll() + return first + } + blocker.cancel() + + #expect(finished) + } } diff --git a/TableProTests/Plugins/JSONLineBatchesTests.swift b/TableProTests/Plugins/JSONLineBatchesTests.swift new file mode 100644 index 000000000..227f16482 --- /dev/null +++ b/TableProTests/Plugins/JSONLineBatchesTests.swift @@ -0,0 +1,78 @@ +// +// JSONLineBatchesTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct JSONLineBatchesTests { + private func write(_ lines: [String]) throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("json-line-batches-\(UUID().uuidString).ndjson") + try Data(lines.map { $0 + "\n" }.joined().utf8).write(to: url) + return url + } + + private func batches(over url: URL, skipsUnreadableLines: Bool = true) throws -> JSONLineBatches { + JSONLineBatches( + lines: try JSONLineReader(url: url), + linesPerBatch: 500, + skipsUnreadableLines: skipsUnreadableLines, + maxRecordedErrors: 1_000 + ) + } + + /// The runner checks for a stop between batches. A batch used to end only once it held 500 + /// rows, so a run of lines holding none was read to the end of the reader's chunk, about + /// 55,000 such lines, before a stop was seen. + @Test("A batch ends after 500 lines even when none of them holds a row") + func batchEndsWithinARunOfUnreadableLines() throws { + let url = try write(Array(repeating: "{ this is not json", count: 5_000)) + defer { try? FileManager.default.removeItem(at: url) } + var lines = try batches(over: url) + defer { lines.close() } + + let first = try #require(try lines.next()) + + #expect(first.isEmpty) + #expect(lines.linesRead == 500) + #expect(lines.unreadableLineCount == 500) + } + + @Test("Batches cover every line once, each row under its own line number") + func batchesCoverEveryLine() throws { + let contents = (1...1_200).map { $0.isMultiple(of: 3) ? "{ this is not json" : #"{"n":\#($0)}"# } + let url = try write(contents) + defer { try? FileManager.default.removeItem(at: url) } + var lines = try batches(over: url) + defer { lines.close() } + + var batchCount = 0 + var entries: [RowImportRunner.Entry] = [] + while let batch = try lines.next() { + batchCount += 1 + entries.append(contentsOf: batch) + } + + #expect(batchCount == 3) + #expect(lines.linesRead == 1_200) + #expect(entries.count == 800) + #expect(lines.unreadableLineCount == 400) + #expect(entries.allSatisfy { $0.row["n"] == .text(String($0.line)) }) + } + + @Test("A stop mode throws at the first unreadable line") + func stopModeThrows() throws { + let url = try write([#"{"n":1}"#, "{ this is not json", #"{"n":3}"#]) + defer { try? FileManager.default.removeItem(at: url) } + var lines = try batches(over: url, skipsUnreadableLines: false) + defer { lines.close() } + + #expect(throws: (any Error).self) { + _ = try lines.next() + } + #expect(lines.linesRead == 2) + } +} diff --git a/TableProTests/Plugins/JSONLineReaderTests.swift b/TableProTests/Plugins/JSONLineReaderTests.swift new file mode 100644 index 000000000..fa8860df6 --- /dev/null +++ b/TableProTests/Plugins/JSONLineReaderTests.swift @@ -0,0 +1,106 @@ +// +// JSONLineReaderTests.swift +// TableProTests +// + +import Foundation +import Testing + +struct JSONLineReaderTests { + private func write(_ bytes: Data) throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("json-line-reader-\(UUID().uuidString).ndjson") + try bytes.write(to: url) + return url + } + + private func readLines(of bytes: Data, chunkSize: Int) throws -> [Data] { + let url = try write(bytes) + defer { try? FileManager.default.removeItem(at: url) } + var reader = try JSONLineReader(url: url, chunkSize: chunkSize) + defer { reader.close() } + var lines: [Data] = [] + while let line = try reader.next() { + lines.append(Data(line)) + } + return lines + } + + private func text(_ lines: [Data]) -> [String?] { + lines.map { String(data: $0, encoding: .utf8) } + } + + @Test("Lines split on the newline byte, with and without a final newline") + func splitsOnNewline() throws { + for chunkSize in [1, 3, 64, JSONLineReader.defaultChunkSize] { + #expect(try text(readLines(of: Data("a\nbb\nccc\n".utf8), chunkSize: chunkSize)) == ["a", "bb", "ccc"]) + #expect(try text(readLines(of: Data("a\nbb\nccc".utf8), chunkSize: chunkSize)) == ["a", "bb", "ccc"]) + } + } + + @Test("An empty file has no lines and a blank line is still a line") + func emptyAndBlankLines() throws { + #expect(try readLines(of: Data(), chunkSize: 4).isEmpty) + #expect(try text(readLines(of: Data("a\n\n\nb\n".utf8), chunkSize: 2)) == ["a", "", "", "b"]) + } + + @Test("Line numbers count blank lines") + func lineNumbersCountBlankLines() throws { + let url = try write(Data("a\n\nb\n".utf8)) + defer { try? FileManager.default.removeItem(at: url) } + var reader = try JSONLineReader(url: url, chunkSize: 2) + defer { reader.close() } + var numbered: [Int: String] = [:] + while let line = try reader.next() { + numbered[reader.lineNumber] = String(data: Data(line), encoding: .utf8) + } + #expect(numbered == [1: "a", 2: "", 3: "b"]) + } + + @Test("A carriage return before the newline stays part of the line") + func carriageReturnStaysInTheLine() throws { + #expect(try text(readLines(of: Data("a\r\nb\r\n".utf8), chunkSize: 3)) == ["a\r", "b\r"]) + } + + /// JSON allows these unescaped inside a string. `URL.lines` ends a line at each of them, + /// which cut a valid row in two. + @Test("U+2028, U+2029 and U+0085 do not end a line") + func unicodeSeparatorsDoNotEndALine() throws { + let line = "{\"a\":\"x\u{2028}y\u{2029}z\u{0085}w\"}" + #expect(try text(readLines(of: Data("\(line)\n{}\n".utf8), chunkSize: 5)) == [line, "{}"]) + } + + @Test("A multi-byte character that straddles a chunk boundary reads back whole") + func multiByteCharacterAcrossChunks() throws { + let line = "{\"tên\":\"Nguyễn 東京\"}" + let bytes = Data("\(line)\n\(line)\n".utf8) + for chunkSize in 1...12 { + #expect(try text(readLines(of: bytes, chunkSize: chunkSize)) == [line, line]) + } + } + + @Test("A line many chunks long reads back whole") + func lineLongerThanManyChunks() throws { + let long = String(repeating: "東", count: 10_000) + let lines = try text(readLines(of: Data("\(long)\nshort\n".utf8), chunkSize: 7)) + #expect(lines == [long, "short"]) + } + + /// Detection checked for a stop between lines only, so closing the sheet over a file with no + /// newline read the whole file into one line first. + @Test("A stop is checked before every chunk, so a line with no end is abandoned") + func stopIsCheckedBeforeEveryChunk() throws { + let url = try write(Data(String(repeating: "x", count: 1_000).utf8)) + defer { try? FileManager.default.removeItem(at: url) } + var checks = 0 + var reader = try JSONLineReader(url: url, chunkSize: 10) { + checks += 1 + if checks > 3 { throw CancellationError() } + } + defer { reader.close() } + #expect(throws: CancellationError.self) { + _ = try reader.next() + } + #expect(checks == 4) + } +} diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index fcff6a221..2369d0119 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -331,6 +331,8 @@ The sheet accepts an array of objects `[{…}, {…}]`, newline-delimited JSON s - **Existing table**: map each JSON field to a column, as [Map fields to columns](#map-fields-to-columns) describes. A column with no field mapped to it keeps its default or NULL. The list holds tables and partitioned tables, and the server routes each row to the right partition. - **New table**: the name field opens on a name derived from the file, already selected, so one keystroke replaces it. Review the inferred columns underneath. Name, type, primary key, nullable flag, and default are all editable before the table is created. +Both lists come from every row in the file. A key that only the last rows carry is still offered, and an inferred type fits every value the field holds, so a field that is a number in all rows but one is typed as text. + The proposed name drops the extension, turns spaces and punctuation into underscores, and lowercases the result. Letters from any script are kept as they are. On Oracle the name comes through in upper case instead, and is cut to 30 bytes rather than 63. A name an existing table or view already holds gains a numeric suffix, so re-importing `users.csv` next to a `users` table proposes `users_2`. Whatever you type over it is held to the same rules: a name already taken, one longer than the engine allows, or one starting with a prefix the engine keeps for itself is reported in the sheet, with **Import** off until it changes. The table list, the new table and the rows all belong to the database and schema the connection was on when the sheet opened. Switching database in another window while the sheet is open leaves the import where it was. diff --git a/project.yml b/project.yml index f6da374f8..d3abf8918 100644 --- a/project.yml +++ b/project.yml @@ -543,10 +543,14 @@ targets: - Plugins/ParquetExportPlugin/ParquetExportModels.swift - Plugins/ParquetExportPlugin/ParquetTableFiles.swift - Plugins/ParquetExportPlugin/ParquetTypeMapper.swift + - Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift + - Plugins/JSONImportPlugin/JSONFieldSurvey.swift - Plugins/JSONImportPlugin/JSONImportOptions.swift - Plugins/JSONImportPlugin/JSONImportOptionsView.swift - Plugins/JSONImportPlugin/JSONImportParsing.swift - Plugins/JSONImportPlugin/JSONImportPlugin.swift + - Plugins/JSONImportPlugin/JSONLineBatches.swift + - Plugins/JSONImportPlugin/JSONLineReader.swift - Plugins/MSSQLDriverPlugin/MSSQLObjectQueries.swift - Plugins/MSSQLDriverPlugin/MSSQLTypeQueries.swift - Plugins/MSSQLDriverPlugin/MSSQLTypeDefinition.swift