fix(plugins): detect JSON import fields from every row and read JSON Lines by byte

CHANGELOG.md

@@ -66,6 +66,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Undo and Redo in a tab with unsaved edits replaying another tab's changes against the wrong rows.
 - Import sheet creating, emptying or filling tables in another database after a database switch in another window.
 - Import sheet discarding a new table's column edits when a CSV or Excel option changes.
+- JSON import leaving out fields first seen after row 200, and typing columns from those rows alone.
+- No fields found in a JSON Lines file with CRLF line endings.
+- JSON Lines rows with U+2028, U+2029 or U+0085 in a string failing to import, and invalid UTF-8 imported as U+FFFD.
+- JSON Lines import stalling while GitHub Copilot is enabled.
 - Table Transfer emptying a destination table, then failing, when two source columns map to one column.
 - MongoDB, Elasticsearch, Typesense and SurrealDB saving a long array or object shortened for display as the cut text.
 - MongoDB refusing text like `[DRAFT] Chapter one...` as a value shortened for display.

Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift

@@ -0,0 +1,52 @@
+//
+//  JSONFieldDetectionCache.swift
+//  JSONImportPlugin
+//
+
+import Foundation
+import os
+import TableProPluginKit
+
+/// The fields of the last JSON Lines file read, kept against that file's identity.
+///
+/// A JSON Lines file's fields do not depend on the table they are matched against, and reading
+/// them means reading every line. The import sheet asks again on every destination table pick, so
+/// each pick used to read the whole file again.
+final class JSONFieldDetectionCache: Sendable {
+    /// A file counts as unchanged while its path, file number, size and modification date all
+    /// match. The identity is taken before the read, so an edit made during one leaves a stale
+    /// identity behind and the next request reads the file again. It describes the file a link
+    /// points to, because `attributesOfItem` describes the link itself and the read follows it.
+    private struct FileIdentity: Equatable, Sendable {
+        let path: String
+        let fileNumber: UInt64?
+        let size: UInt64?
+        let modificationDate: Date?
+
+        init(of url: URL) throws {
+            let file = url.resolvingSymlinksInPath()
+            let attributes = try FileManager.default.attributesOfItem(atPath: file.path)
+            path = file.path
+            fileNumber = (attributes[.systemFileNumber] as? NSNumber)?.uint64Value
+            size = (attributes[.size] as? NSNumber)?.uint64Value
+            modificationDate = attributes[.modificationDate] as? Date
+        }
+    }
+
+    private struct Entry: Sendable {
+        let file: FileIdentity
+        let fields: [PluginImportField]
+    }
+
+    private let lastEntry = OSAllocatedUnfairLock<Entry?>(initialState: nil)
+
+    func fields(at url: URL, detect: () throws -> [PluginImportField]) throws -> [PluginImportField] {
+        let file = try FileIdentity(of: url)
+        if let cached = lastEntry.withLock({ $0?.file == file ? $0?.fields : nil }) {
+            return cached
+        }
+        let fields = try detect()
+        lastEntry.withLock { $0 = Entry(file: file, fields: fields) }
+        return fields
+    }
+}

Plugins/JSONImportPlugin/JSONFieldSurvey.swift

@@ -0,0 +1,102 @@
+//
+//  JSONFieldSurvey.swift
+//  JSONImportPlugin
+//
+
+import Foundation
+import TableProPluginKit
+
+/// Every field any row of the file names, each with the first value it holds and a type that fits
+/// every value it holds. It keeps one small record per field rather than the values themselves,
+/// so it can read every row of a file of any size.
+struct JSONFieldSurvey {
+    private var profiles: [String: JSONFieldProfile] = [:]
+
+    mutating func add(_ row: NSDictionary) {
+        row.enumerateKeysAndObjects { key, value, _ in
+            guard let name = key as? String else { return }
+            profiles[name, default: JSONFieldProfile()].add(value)
+        }
+    }
+
+    var fields: [PluginImportField] {
+        profiles.keys.sorted().map { name in
+            let profile = profiles[name] ?? JSONFieldProfile()
+            return PluginImportField(
+                name: name,
+                sampleValue: profile.sampleValue,
+                inferredType: profile.kinds.inferredType
+            )
+        }
+    }
+}
+
+struct JSONFieldProfile {
+    private(set) var sampleValue: String?
+    private(set) var kinds = JSONValueKinds()
+
+    mutating func add(_ value: Any) {
+        let kind = JSONValueKind(of: value)
+        guard kind != .null else { return }
+        if sampleValue == nil {
+            sampleValue = JSONImportParsing.sampleString(value)
+        }
+        kinds.add(kind)
+    }
+}
+
+struct JSONValueKinds {
+    private var sawValue = false
+    private var allNested = true
+    private var allBoolean = true
+    private var allInteger = true
+    private var allNumber = true
+
+    mutating func add(_ kind: JSONValueKind) {
+        sawValue = true
+        if kind != .nested { allNested = false }
+        if kind != .boolean { allBoolean = false }
+        if kind != .integer { allInteger = false }
+        if kind != .integer, kind != .real { allNumber = false }
+    }
+
+    var inferredType: PluginImportFieldType {
+        guard sawValue else { return .text }
+        if allNested { return .json }
+        if allBoolean { return .boolean }
+        if allInteger { return .integer }
+        if allNumber { return .real }
+        return .text
+    }
+}
+
+/// Reads a parsed JSON value's kind from its Core Foundation type. A survey classifies every value
+/// in the file, and a Swift `is` or `as?` cast from `Any` costs about ten times as much.
+enum JSONValueKind: Equatable {
+    case null
+    case nested
+    case boolean
+    case integer
+    case real
+    case other
+
+    init(of value: Any) {
+        let object = value as AnyObject
+        switch CFGetTypeID(object) {
+        case CFNullGetTypeID():
+            self = .null
+        case CFArrayGetTypeID(), CFDictionaryGetTypeID():
+            self = .nested
+        case CFBooleanGetTypeID():
+            self = .boolean
+        case CFNumberGetTypeID():
+            guard let number = object as? NSNumber else {
+                self = .other
+                return
+            }
+            self = CFNumberIsFloatType(number) ? .real : .integer
+        default:
+            self = .other
+        }
+    }
+}

Plugins/JSONImportPlugin/JSONImportParsing.swift

@@ -16,23 +16,32 @@ enum JSONImportParsing {
         ["jsonl", "ndjson"].contains(url.pathExtension.lowercased())
     }
 
-    static func parseRow(fromLine line: String) throws -> [String: PluginCellValue] {
-        let object = try JSONSerialization.jsonObject(with: Data(line.utf8))
-        guard let dict = object as? [String: Any] else {
+    static func parseRow(fromLine line: Data) throws -> [String: PluginCellValue]? {
+        try object(fromLine: line).map(convertRow)
+    }
+
+    static func object(fromLine line: Data) throws -> NSDictionary? {
+        guard !isBlank(line) else { return nil }
+        let object = try JSONSerialization.jsonObject(with: line)
+        guard let dict = object as? NSDictionary else {
             throw PluginImportError.importFailed("Each line must be a JSON object")
         }
-        return convertRow(dict)
+        return dict
+    }
+
+    private static func isBlank(_ line: Data) -> Bool {
+        line.allSatisfy { $0 == 0x20 || $0 == 0x09 || $0 == 0x0D }
     }
 
-    static func parseRows(at url: URL, targetTable: String?) throws -> [[String: Any]] {
+    static func parseRows(at url: URL, targetTable: String?) throws -> [NSDictionary] {
         let data = try Data(contentsOf: url)
         let object = try JSONSerialization.jsonObject(with: data)
         return try extractRows(from: object, targetTable: targetTable)
     }
 
-    static func extractRows(from object: Any, targetTable: String?) throws -> [[String: Any]] {
+    static func extractRows(from object: Any, targetTable: String?) throws -> [NSDictionary] {
         if let array = object as? [Any] {
-            return array.compactMap { $0 as? [String: Any] }
+            return array.compactMap { $0 as? NSDictionary }
         }
 
         guard let dict = object as? [String: Any] else {
@@ -41,19 +50,19 @@ enum JSONImportParsing {
 
         let tables = dict.compactMapValues { value -> [Any]? in
             guard let array = value as? [Any] else { return nil }
-            return array.allSatisfy { $0 is [String: Any] } ? array : nil
+            return array.allSatisfy { $0 is NSDictionary } ? array : nil
         }
         let isTableWrapper = !tables.isEmpty && tables.count == dict.count
 
         guard isTableWrapper else {
-            return [dict]
+            return [dict as NSDictionary]
         }
 
         if let targetTable, let match = matchTable(in: tables, to: targetTable) {
-            return match.compactMap { $0 as? [String: Any] }
+            return match.compactMap { $0 as? NSDictionary }
         }
         if tables.count == 1, let only = tables.values.first {
-            return only.compactMap { $0 as? [String: Any] }
+            return only.compactMap { $0 as? NSDictionary }
         }
         throw PluginImportError.importFailed("The file contains multiple tables and none matches the target table")
     }
@@ -68,8 +77,14 @@ enum JSONImportParsing {
         return suffix?.value
     }
 
-    static func convertRow(_ row: [String: Any]) -> [String: PluginCellValue] {
-        row.mapValues(cellValue(from:))
+    static func convertRow(_ row: NSDictionary) -> [String: PluginCellValue] {
+        var converted: [String: PluginCellValue] = [:]
+        converted.reserveCapacity(row.count)
+        row.enumerateKeysAndObjects { key, value, _ in
+            guard let name = key as? String else { return }
+            converted[name] = cellValue(from: value)
+        }
+        return converted
     }
 
     static func cellValue(from json: Any) -> PluginCellValue {
@@ -94,94 +109,37 @@ enum JSONImportParsing {
 
     // MARK: - Source introspection
 
-    static let sampleLength = 256 * 1_024
-
-    private static let longestUTF8Tail = 3
-
-    static func utf8Text(ofSample sample: Data) -> String {
-        for dropped in 0...min(longestUTF8Tail, sample.count) {
-            if let text = String(data: sample.dropLast(dropped), encoding: .utf8) {
-                return text
-            }
+    static func detectFields(at url: URL, targetTable: String?) throws -> [PluginImportField] {
+        try Task.checkCancellation()
+        guard isLineDelimited(url) else {
+            return try detectFields(in: try parseRows(at: url, targetTable: targetTable))
         }
-        return ""
+        return try detectFields(inLinesAt: url)
     }
 
-    static func sampleRawRows(at url: URL, targetTable: String?, limit: Int) throws -> [[String: Any]] {
-        if isLineDelimited(url) {
-            let handle = try FileHandle(forReadingFrom: url)
-            defer { try? handle.close() }
-            let text = utf8Text(ofSample: handle.readData(ofLength: sampleLength))
-            var rows: [[String: Any]] = []
-            for line in text.split(separator: "\n") where rows.count < limit {
-                let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines)
-                guard !trimmed.isEmpty else { continue }
-                if let object = try? JSONSerialization.jsonObject(with: Data(trimmed.utf8)) as? [String: Any] {
-                    rows.append(object)
-                }
+    static func detectFields(inLinesAt url: URL) throws -> [PluginImportField] {
+        var survey = JSONFieldSurvey()
+        var lines = try JSONLineReader(url: url)
+        defer { lines.close() }
+        while let line = try lines.next() {
+            autoreleasepool {
+                guard let row = try? object(fromLine: line) else { return }
+                survey.add(row)
             }
-            return rows
         }
-        let object = try JSONSerialization.jsonObject(with: Data(contentsOf: url))
-        return Array(try extractRows(from: object, targetTable: targetTable).prefix(limit))
+        return survey.fields
     }
 
-    static func detectFields(in rows: [[String: Any]]) -> [PluginImportField] {
-        var names: [String] = []
-        var seen = Set<String>()
-        var valuesByField: [String: [Any]] = [:]
+    static func detectFields(in rows: [NSDictionary]) throws -> [PluginImportField] {
+        var survey = JSONFieldSurvey()
         for row in rows {
-            for (key, value) in row {
-                if seen.insert(key).inserted { names.append(key) }
-                valuesByField[key, default: []].append(value)
-            }
-        }
-        return names.sorted().map { name in
-            let nonNull = (valuesByField[name] ?? []).filter { !($0 is NSNull) }
-            return PluginImportField(
-                name: name,
-                sampleValue: nonNull.first.map(sampleString),
-                inferredType: inferType(from: nonNull)
-            )
-        }
-    }
-
-    static func inferType(from values: [Any]) -> PluginImportFieldType {
-        guard !values.isEmpty else { return .text }
-        var allNested = true
-        var allBoolean = true
-        var allInteger = true
-        var allNumber = true
-        for value in values {
-            if value is [Any] || value is [String: Any] {
-                allBoolean = false
-                allInteger = false
-                allNumber = false
-            } else {
-                allNested = false
-                if let number = value as? NSNumber {
-                    if CFGetTypeID(number) == CFBooleanGetTypeID() {
-                        allInteger = false
-                        allNumber = false
-                    } else {
-                        allBoolean = false
-                        if CFNumberIsFloatType(number) { allInteger = false }
-                    }
-                } else {
-                    allBoolean = false
-                    allInteger = false
-                    allNumber = false
-                }
-            }
+            try Task.checkCancellation()
+            survey.add(row)
         }
-        if allNested { return .json }
-        if allBoolean { return .boolean }
-        if allInteger { return .integer }
-        if allNumber { return .real }
-        return .text
+        return survey.fields
     }
 
-    private static func sampleString(_ value: Any) -> String {
+    static func sampleString(_ value: Any) -> String {
         switch cellValue(from: value) {
         case .text(let string): return String(string.prefix(80))
         case .bytes, .null: return ""