diff --git a/CHANGELOG.md b/CHANGELOG.md index 29b0b8a0e..3ec853252 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - iCloud sync mixing up two favorite tables whose names contain a vertical bar. - iCloud sync sending both a save and a deletion for an item unstarred and starred again, or renamed back, before it ran. - Favorite table starred again while its removal was syncing to iCloud disappearing when the sync finished. +- Table favorites, saved queries and column layouts of **Local only** connections syncing to iCloud. +- Database favorites and column layouts removed on another Mac staying on this one. ## [0.76.1] - 2026-09-29 diff --git a/Packages/TableProCore/Sources/TableProSyncTransport/SyncMetadataStorage.swift b/Packages/TableProCore/Sources/TableProSyncTransport/SyncMetadataStorage.swift index d615f5f17..698c525ef 100644 --- a/Packages/TableProCore/Sources/TableProSyncTransport/SyncMetadataStorage.swift +++ b/Packages/TableProCore/Sources/TableProSyncTransport/SyncMetadataStorage.swift @@ -2,13 +2,15 @@ import CloudKit import Foundation import os -public struct Tombstone: Codable, Sendable { +public struct Tombstone: Codable, Equatable, Sendable { public let id: String public let deletedAt: Date + public let owner: UUID? - public init(id: String, deletedAt: Date = Date()) { + public init(id: String, deletedAt: Date = Date(), owner: UUID? = nil) { self.id = id self.deletedAt = deletedAt + self.owner = owner } } @@ -108,11 +110,13 @@ public final class SyncMetadataStorage: @unchecked Sendable { addTombstones([id], type: type) } - public func addTombstones(_ ids: [String], type: SyncRecordType) { - guard !ids.isEmpty else { return } - var current = tombstones(for: type) - current.append(contentsOf: ids.map { Tombstone(id: $0) }) - saveTombstones(current, for: type) + public func addTombstones(_ ids: [String], type: SyncRecordType, owner: UUID? = nil) { + addTombstones(ids.map { Tombstone(id: $0, owner: owner) }, type: type) + } + + public func addTombstones(_ added: [Tombstone], type: SyncRecordType) { + guard !added.isEmpty else { return } + saveTombstones(tombstones(for: type) + added, for: type) } public func removeTombstone(_ id: String, type: SyncRecordType) { @@ -125,17 +129,50 @@ public final class SyncMetadataStorage: @unchecked Sendable { userDefaults.removeObject(forKey: tombstoneKey(type)) } - public func pruneTombstones(olderThan days: Int) { + public func pruneTombstones( + olderThan days: Int, + where isPushable: (SyncRecordType, Tombstone) -> Bool + ) { let cutoff = Calendar.current.date(byAdding: .day, value: -days, to: Date()) ?? Date() + removeTombstones { type, tombstone in + tombstone.deletedAt < cutoff && isPushable(type, tombstone) + } + } + + public func removeTombstones(where shouldRemove: (SyncRecordType, Tombstone) -> Bool) { for type in SyncRecordType.allCases { var current = tombstones(for: type) let before = current.count - current.removeAll { $0.deletedAt < cutoff } + current.removeAll { shouldRemove(type, $0) } guard current.count != before else { continue } saveTombstones(current, for: type) } } + // MARK: - Owners Kept Off Sync + + public func ownersKeptOffSync() -> Set { + Set((userDefaults.stringArray(forKey: key("ownersKeptOffSync")) ?? []).compactMap(UUID.init(uuidString:))) + } + + public func keepOffSync(owners: Set) { + guard !owners.isEmpty else { return } + saveOwnersKeptOffSync(ownersKeptOffSync().union(owners)) + } + + public func releaseOwnersKeptOffSync(_ owners: Set) { + guard !owners.isEmpty else { return } + saveOwnersKeptOffSync(ownersKeptOffSync().subtracting(owners)) + } + + private func saveOwnersKeptOffSync(_ owners: Set) { + guard !owners.isEmpty else { + userDefaults.removeObject(forKey: key("ownersKeptOffSync")) + return + } + userDefaults.set(owners.map(\.uuidString).sorted(), forKey: key("ownersKeptOffSync")) + } + // MARK: - Last Sync Date public var lastSyncDate: Date? { diff --git a/Packages/TableProCore/Tests/TableProSyncTests/SyncMetadataStorageTests.swift b/Packages/TableProCore/Tests/TableProSyncTests/SyncMetadataStorageTests.swift index eaf9a0d9c..abfaf9dea 100644 --- a/Packages/TableProCore/Tests/TableProSyncTests/SyncMetadataStorageTests.swift +++ b/Packages/TableProCore/Tests/TableProSyncTests/SyncMetadataStorageTests.swift @@ -68,11 +68,82 @@ struct SyncMetadataStorageTests { defaults.set(data, forKey: "com.TablePro.sync.tombstones.\(SyncRecordType.connection.rawValue)") let storage = SyncMetadataStorage(userDefaults: defaults) - storage.pruneTombstones(olderThan: 30) + storage.pruneTombstones(olderThan: 30) { _, _ in true } #expect(storage.tombstones(for: .connection).map(\.id) == ["fresh"]) } + @Test("Pruning keeps an old tombstone that cannot be pushed yet") + func pruningKeepsUnpushableTombstones() throws { + let defaults = UserDefaults(suiteName: "com.TablePro.tests.\(UUID().uuidString)") ?? .standard + let heldOwner = UUID() + let fortyDaysAgo = Date(timeIntervalSinceNow: -60 * 60 * 24 * 40) + let held = Tombstone(id: "held", deletedAt: fortyDaysAgo, owner: heldOwner) + let released = Tombstone(id: "released", deletedAt: fortyDaysAgo, owner: UUID()) + let data = try JSONEncoder().encode([held, released]) + defaults.set(data, forKey: "com.TablePro.sync.tombstones.\(SyncRecordType.tableFavorite.rawValue)") + + let storage = SyncMetadataStorage(userDefaults: defaults) + storage.pruneTombstones(olderThan: 30) { _, tombstone in tombstone.owner != heldOwner } + + #expect(storage.tombstones(for: .tableFavorite).map(\.id) == ["held"]) + } + + @Test("A tombstone keeps the owner it was recorded with") + func tombstoneOwnerRoundTrips() { + let storage = makeStorage() + let owner = UUID() + storage.addTombstones(["a", "b"], type: .tableFavorite, owner: owner) + storage.addTombstone("c", type: .tableFavorite) + + #expect(storage.tombstones(for: .tableFavorite).map(\.owner) == [owner, owner, nil]) + } + + @Test("A tombstone written before owners existed still decodes, with no owner") + func legacyTombstoneDecodesWithoutOwner() throws { + let defaults = UserDefaults(suiteName: "com.TablePro.tests.\(UUID().uuidString)") ?? .standard + let legacy = Data(#"[{"id":"a","deletedAt":780000000}]"#.utf8) + defaults.set(legacy, forKey: "com.TablePro.sync.tombstones.\(SyncRecordType.favoriteDatabase.rawValue)") + + let tombstones = SyncMetadataStorage(userDefaults: defaults).tombstones(for: .favoriteDatabase) + + #expect(tombstones.map(\.id) == ["a"]) + #expect(tombstones.map(\.owner) == [nil]) + } + + @Test("Owners kept off sync survive a new storage instance and can be released one by one") + func ownersKeptOffSyncPersist() { + let defaults = UserDefaults(suiteName: "com.TablePro.tests.\(UUID().uuidString)") ?? .standard + let first = UUID() + let second = UUID() + SyncMetadataStorage(userDefaults: defaults).keepOffSync(owners: [first, second]) + + let storage = SyncMetadataStorage(userDefaults: defaults) + #expect(storage.ownersKeptOffSync() == [first, second]) + + storage.releaseOwnersKeptOffSync([first]) + #expect(storage.ownersKeptOffSync() == [second]) + storage.clearAll() + #expect(storage.ownersKeptOffSync() == [second]) + } + + @Test("Removing tombstones by a predicate reaches every type and leaves the rest") + func removingOwnedTombstonesKeepsTheRest() { + let storage = makeStorage() + let removed = UUID() + let kept = UUID() + storage.addTombstones(["a"], type: .tableFavorite, owner: removed) + storage.addTombstones(["b"], type: .favorite, owner: removed) + storage.addTombstones(["c"], type: .tableFavorite, owner: kept) + storage.addTombstone("d", type: .tag) + + storage.removeTombstones { _, tombstone in tombstone.owner == removed } + + #expect(storage.tombstones(for: .tableFavorite).map(\.id) == ["c"]) + #expect(storage.tombstones(for: .favorite).isEmpty) + #expect(storage.tombstones(for: .tag).map(\.id) == ["d"]) + } + @Test("The last sync date round-trips") func lastSyncDateRoundTrips() { let storage = makeStorage() diff --git a/TablePro/Core/Storage/ColumnLayoutPersister.swift b/TablePro/Core/Storage/ColumnLayoutPersister.swift index 97fa333e5..e0523c631 100644 --- a/TablePro/Core/Storage/ColumnLayoutPersister.swift +++ b/TablePro/Core/Storage/ColumnLayoutPersister.swift @@ -27,7 +27,7 @@ final class FileColumnLayoutPersister: ColumnLayoutPersisting, TableScopedSettin var hiddenColumns: [String]? } - static let syncCategoryPrefix = "columnLayout." + nonisolated static let syncCategoryPrefix = "columnLayout." private let storageDirectory: URL private let defaults: UserDefaults @@ -141,7 +141,7 @@ final class FileColumnLayoutPersister: ColumnLayoutPersisting, TableScopedSettin cache[oldScope.connectionId] = entries writeEntries(entries, for: oldScope.connectionId) syncTracker.markDirty(.settings, id: Self.syncCategory(for: newKey)) - syncTracker.markDeleted(.settings, id: Self.syncCategory(for: oldKey)) + syncTracker.markDeleted(.settings, ids: [Self.syncCategory(for: oldKey)], owner: oldScope.connectionId) } /// Moves every table's saved layout from one container to another. Same prefix rewrite as the @@ -173,7 +173,7 @@ final class FileColumnLayoutPersister: ColumnLayoutPersisting, TableScopedSettin writeEntries(entries, for: connectionId) for key in moving { syncTracker.markDirty(.settings, id: Self.syncCategory(for: newPrefix + key.dropFirst(oldPrefix.count))) - syncTracker.markDeleted(.settings, id: Self.syncCategory(for: key)) + syncTracker.markDeleted(.settings, ids: [Self.syncCategory(for: key)], owner: connectionId) } } @@ -199,20 +199,14 @@ final class FileColumnLayoutPersister: ColumnLayoutPersisting, TableScopedSettin entries.removeValue(forKey: key) } - if entries.isEmpty { - cache[connectionId] = [:] - removeFile(for: connectionId) - } else { - cache[connectionId] = entries - writeEntries(entries, for: connectionId) - } - syncTracker.markDeleted(.settings, ids: dropping.map(Self.syncCategory(for:))) + guard store(entries, for: connectionId) else { return } + syncTracker.markDeleted(.settings, ids: dropping.map(Self.syncCategory(for:)), owner: connectionId) } func purgeConnections(_ connectionIds: Set, leavesTombstones: Bool) { - var categories: [String] = [] + var categoriesByConnection: [UUID: Set] = [:] for connectionId in connectionIds { - categories += loadEntries(for: connectionId).keys.map(Self.syncCategory(for:)) + categoriesByConnection[connectionId] = Set(loadEntries(for: connectionId).keys.map(Self.syncCategory(for:))) cache[connectionId] = [:] removeFile(for: connectionId) } @@ -220,9 +214,9 @@ final class FileColumnLayoutPersister: ColumnLayoutPersisting, TableScopedSettin /// own deletion back at it, but leaving the ids dirty means the next push looks for entries /// that are gone and never drains them. if leavesTombstones { - syncTracker.markDeleted(.settings, ids: categories) + syncTracker.markDeleted(.settings, idsByOwner: categoriesByConnection) } else { - syncTracker.discardDirty(.settings, ids: categories) + syncTracker.discardDirty(.settings, ids: categoriesByConnection.values.flatMap { $0 }) } } @@ -232,14 +226,8 @@ final class FileColumnLayoutPersister: ColumnLayoutPersisting, TableScopedSettin var entries = loadEntries(for: key.connectionId) guard entries.removeValue(forKey: key.storageKey) != nil else { return } - if entries.isEmpty { - cache[key.connectionId] = [:] - removeFile(for: key.connectionId) - } else { - cache[key.connectionId] = entries - writeEntries(entries, for: key.connectionId) - } - syncTracker.markDeleted(.settings, id: Self.syncCategory(for: key.storageKey)) + guard store(entries, for: key.connectionId) else { return } + syncTracker.markDeleted(.settings, ids: [Self.syncCategory(for: key.storageKey)], owner: key.connectionId) } func clearGeometry(for key: ColumnLayoutTableKey) { @@ -256,14 +244,8 @@ final class FileColumnLayoutPersister: ColumnLayoutPersisting, TableScopedSettin syncTracker.markDirty(.settings, id: Self.syncCategory(for: key.storageKey)) } else { entries.removeValue(forKey: key.storageKey) - if entries.isEmpty { - cache[key.connectionId] = [:] - removeFile(for: key.connectionId) - } else { - cache[key.connectionId] = entries - writeEntries(entries, for: key.connectionId) - } - syncTracker.markDeleted(.settings, id: Self.syncCategory(for: key.storageKey)) + guard store(entries, for: key.connectionId) else { return } + syncTracker.markDeleted(.settings, ids: [Self.syncCategory(for: key.storageKey)], owner: key.connectionId) } } @@ -271,6 +253,39 @@ final class FileColumnLayoutPersister: ColumnLayoutPersisting, TableScopedSettin syncCategoryPrefix + storageKey } + nonisolated static func connectionId(ofSyncCategory category: String) -> UUID? { + guard category.hasPrefix(syncCategoryPrefix) else { return nil } + return TableScope(storageComponent: String(category.dropFirst(syncCategoryPrefix.count)))?.connectionId + } + + func storageKeys(forSyncRecordNames recordNames: Set) -> [String] { + let layoutPrefix = SyncRecordType.settings.recordNamePrefix + Self.syncCategoryPrefix + let digestPrefix = SyncRecordType.settings.recordNamePrefix + SyncRecordName.digestPrefix + let named = recordNames.filter { $0.hasPrefix(layoutPrefix) }.map { String($0.dropFirst(layoutPrefix.count)) } + let digested = recordNames.filter { $0.hasPrefix(digestPrefix) } + guard !digested.isEmpty else { return named } + return named + customizedStorageKeys().filter { storageKey in + digested.contains(SyncRecordType.settings.recordName(for: Self.syncCategory(for: storageKey))) + } + } + + func removeWithoutSync(storageKeys: [String]) -> Bool { + let scoped = storageKeys.compactMap { key in TableScope(storageComponent: key).map { (key, $0.connectionId) } } + var persisted = true + for (connectionId, keys) in Dictionary(grouping: scoped, by: \.1) { + var entries = loadEntries(for: connectionId) + let removed = keys.map(\.0).filter { entries.removeValue(forKey: $0) != nil } + guard !removed.isEmpty else { continue } + guard store(entries, for: connectionId) else { + persisted = false + continue + } + removed.forEach(removeLegacyHidden(storageKey:)) + syncTracker.discardDirty(.settings, ids: removed.map(Self.syncCategory(for:))) + } + return persisted + } + func rawData(forStorageKey storageKey: String) -> Data? { guard let scope = TableScope(storageComponent: storageKey), let entry = loadEntries(for: scope.connectionId)[storageKey] else { return nil } @@ -309,7 +324,18 @@ final class FileColumnLayoutPersister: ColumnLayoutPersisting, TableScopedSettin } private func removeLegacyHidden(for key: ColumnLayoutTableKey) { - defaults.removeObject(forKey: Self.legacyVisibilityPrefix + key.storageKey) + removeLegacyHidden(storageKey: key.storageKey) + } + + private func removeLegacyHidden(storageKey: String) { + defaults.removeObject(forKey: Self.legacyVisibilityPrefix + storageKey) + } + + @discardableResult + private func store(_ entries: [String: PersistedColumnLayout], for connectionId: UUID) -> Bool { + let stored = entries.isEmpty ? removeFile(for: connectionId) : writeEntries(entries, for: connectionId) + cache[connectionId] = stored ? entries : nil + return stored } private func loadEntries(for connectionId: UUID) -> [String: PersistedColumnLayout] { @@ -335,27 +361,33 @@ final class FileColumnLayoutPersister: ColumnLayoutPersisting, TableScopedSettin } } - private func writeEntries(_ entries: [String: PersistedColumnLayout], for connectionId: UUID) { + @discardableResult + private func writeEntries(_ entries: [String: PersistedColumnLayout], for connectionId: UUID) -> Bool { let fileURL = fileURL(for: connectionId) do { let data = try encoder.encode(entries) try data.write(to: fileURL, options: .atomic) + return true } catch { Self.logger.error( "Failed to write column layouts for \(connectionId): \(error.localizedDescription)" ) + return false } } - private func removeFile(for connectionId: UUID) { + @discardableResult + private func removeFile(for connectionId: UUID) -> Bool { let fileURL = fileURL(for: connectionId) - guard FileManager.default.fileExists(atPath: fileURL.path) else { return } + guard FileManager.default.fileExists(atPath: fileURL.path) else { return true } do { try FileManager.default.removeItem(at: fileURL) + return true } catch { Self.logger.error( "Failed to remove column layout file for \(connectionId): \(error.localizedDescription)" ) + return false } } diff --git a/TablePro/Core/Storage/ConnectionLocalState.swift b/TablePro/Core/Storage/ConnectionLocalState.swift index f6486158b..d38ff0c93 100644 --- a/TablePro/Core/Storage/ConnectionLocalState.swift +++ b/TablePro/Core/Storage/ConnectionLocalState.swift @@ -16,10 +16,16 @@ internal enum ConnectionLocalState { nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "ConnectionLocalState") /// Who deleted the connection. A local delete leaves tombstones so the other devices follow; - /// a remote delete must not, or it pushes back a deletion the sender already made. + /// a remote delete must not, or it pushes back a deletion the sender already made, and neither + /// may the local delete of a connection kept off iCloud. internal enum Origin { case local + case localOnly case remote + + var leavesTombstones: Bool { + self == .local + } } internal static func purge( @@ -31,7 +37,8 @@ internal enum ConnectionLocalState { favoriteDatabases: FavoriteDatabasesStorage = .shared, sqlFavorites: SQLFavoriteManager = .shared, queryHistory: QueryHistoryManager = .shared, - defaults: UserDefaults = AppStorageEnvironment.shared.defaults + defaults: UserDefaults = AppStorageEnvironment.shared.defaults, + syncTracker: SyncChangeTracker = .shared ) { guard !connectionIds.isEmpty else { return } @@ -39,7 +46,12 @@ internal enum ConnectionLocalState { purgeLiveState(connectionId) appSettings.saveLastDatabase(nil, for: connectionId) appSettings.saveLastSchema(nil, for: connectionId) - purgeFavorites(connectionId, origin: origin, tables: favoriteTables, databases: favoriteDatabases) + purgeFavorites( + connectionId, + leavesTombstones: origin.leavesTombstones, + tables: favoriteTables, + databases: favoriteDatabases + ) SidebarPersistenceKey.removeAll(connectionId: connectionId) RecentTablesStore.shared.removeEntries(for: connectionId) HistoryPanelPreferencesStorage.remove(for: connectionId) @@ -49,7 +61,11 @@ internal enum ConnectionLocalState { purgeTrailingPaneKeys(connectionIds, defaults: defaults) for store in tableScopedStores { - store.purgeConnections(connectionIds, leavesTombstones: origin == .local) + store.purgeConnections(connectionIds, leavesTombstones: origin.leavesTombstones) + } + if origin == .localOnly { + syncTracker.discardTombstones(ownedBy: connectionIds) + syncTracker.keepOffSync(owners: connectionIds) } DatabaseTreeFilterStorage.shared.removeFilters(for: connectionIds) LoadableExtensionApprovalStore.shared.revoke(for: connectionIds) @@ -57,7 +73,11 @@ internal enum ConnectionLocalState { WorkspaceRailOrderStore.shared.removeEntries(for: connectionIds) Task { await purgeAsyncStores( - connectionIds, origin: origin, sqlFavorites: sqlFavorites, queryHistory: queryHistory + connectionIds, + origin: origin, + sqlFavorites: sqlFavorites, + queryHistory: queryHistory, + syncTracker: syncTracker ) } } @@ -82,12 +102,17 @@ internal enum ConnectionLocalState { _ connectionIds: Set, origin: Origin, sqlFavorites: SQLFavoriteManager = .shared, - queryHistory: QueryHistoryManager = .shared + queryHistory: QueryHistoryManager = .shared, + syncTracker: SyncChangeTracker = .shared ) async { for connectionId in connectionIds { switch origin { case .local: await sqlFavorites.removeFavoritesAndFolders(for: connectionId) + case .localOnly: + if await sqlFavorites.removeFavoritesAndFoldersWithoutSync(for: connectionId) { + syncTracker.releaseOwnersKeptOffSync([connectionId]) + } case .remote: await sqlFavorites.removeFavoritesAndFoldersWithoutSync(for: connectionId) } @@ -129,17 +154,16 @@ internal enum ConnectionLocalState { private static func purgeFavorites( _ connectionId: UUID, - origin: Origin, + leavesTombstones: Bool, tables: FavoriteTablesStorage, databases: FavoriteDatabasesStorage ) { - switch origin { - case .local: - tables.removeFavorites(for: connectionId) - databases.removeFavorites(for: connectionId) - case .remote: + guard leavesTombstones else { tables.removeFavoritesWithoutSync(for: connectionId) databases.removeFavoritesWithoutSync(for: connectionId) + return } + tables.removeFavorites(for: connectionId) + databases.removeFavorites(for: connectionId) } } diff --git a/TablePro/Core/Storage/ConnectionStorage.swift b/TablePro/Core/Storage/ConnectionStorage.swift index 2435bb371..8a0fafd24 100644 --- a/TablePro/Core/Storage/ConnectionStorage.swift +++ b/TablePro/Core/Storage/ConnectionStorage.swift @@ -147,7 +147,7 @@ final class ConnectionStorage { Self.logger.error("Aborted addConnection: persistence failed for \(connection.id, privacy: .public)") return } - if !connection.localOnly && !connection.isSample { + if connection.participatesInSync { syncTracker.markDirty(.connection, id: connection.id.uuidString) } @@ -165,7 +165,7 @@ final class ConnectionStorage { Self.logger.error("Aborted updateConnection: persistence failed for \(connection.id, privacy: .public)") return } - if !connection.localOnly && !connection.isSample { + if connection.participatesInSync { syncTracker.markDirty(.connection, id: connection.id.uuidString) } @@ -196,7 +196,7 @@ final class ConnectionStorage { return false } let dirtyIds = updatesById.values - .filter { !$0.localOnly && !$0.isSample } + .filter(\.participatesInSync) .map { $0.id.uuidString } syncTracker.markDirty(.connection, ids: dirtyIds) return true @@ -239,7 +239,7 @@ final class ConnectionStorage { return false } let dirtyIds = changed - .filter { !$0.localOnly && !$0.isSample } + .filter(\.participatesInSync) .map { $0.id.uuidString } syncTracker.markDirty(.connection, ids: dirtyIds) appEventsProvider().connectionUpdated.send(changed.count == 1 ? changed.first?.id : nil) @@ -321,7 +321,7 @@ final class ConnectionStorage { } let updatedConnection = connections[index] - if !updatedConnection.localOnly && !updatedConnection.isSample { + if updatedConnection.participatesInSync { syncTracker.markDirty(.connection, id: updatedConnection.id.uuidString) } @@ -337,7 +337,7 @@ final class ConnectionStorage { Self.logger.error("Aborted deleteConnection: persistence failed for \(connection.id, privacy: .public)") return false } - if !connection.localOnly && !connection.isSample { + if connection.participatesInSync { syncTracker.markDeleted(.connection, id: connection.id.uuidString) } deletePassword(for: connection.id) @@ -355,8 +355,9 @@ final class ConnectionStorage { ConnectionLocalState.purge( connectionIds: [connection.id], - origin: .local, - appSettings: appSettingsProvider() + origin: connection.participatesInSync ? .local : .localOnly, + appSettings: appSettingsProvider(), + syncTracker: syncTracker ) return true } @@ -371,7 +372,7 @@ final class ConnectionStorage { Self.logger.error("Aborted deleteConnections: persistence failed for \(idsToDelete.count, privacy: .public) connection(s)") return false } - for conn in connectionsToDelete where !conn.localOnly && !conn.isSample { + for conn in connectionsToDelete where conn.participatesInSync { syncTracker.markDeleted(.connection, id: conn.id.uuidString) } for conn in connectionsToDelete { @@ -387,10 +388,18 @@ final class ConnectionStorage { let fields = Self.secureFieldIds(for: conn.type) deleteAllPluginSecureFields(for: conn.id, fieldIds: fields) } + let syncedIds = Set(connectionsToDelete.filter(\.participatesInSync).map(\.id)) ConnectionLocalState.purge( - connectionIds: idsToDelete, + connectionIds: syncedIds, origin: .local, - appSettings: appSettingsProvider() + appSettings: appSettingsProvider(), + syncTracker: syncTracker + ) + ConnectionLocalState.purge( + connectionIds: idsToDelete.subtracting(syncedIds), + origin: .localOnly, + appSettings: appSettingsProvider(), + syncTracker: syncTracker ) return true } @@ -456,7 +465,7 @@ final class ConnectionStorage { return nil } let dirtyIds = ([placedDuplicate] + renumbered) - .filter { !$0.localOnly && !$0.isSample } + .filter(\.participatesInSync) .map { $0.id.uuidString } syncTracker.markDirty(.connection, ids: dirtyIds) diff --git a/TablePro/Core/Storage/FavoriteDatabasesStorage.swift b/TablePro/Core/Storage/FavoriteDatabasesStorage.swift index f7e9345d7..422335c8a 100644 --- a/TablePro/Core/Storage/FavoriteDatabasesStorage.swift +++ b/TablePro/Core/Storage/FavoriteDatabasesStorage.swift @@ -86,9 +86,10 @@ internal final class FavoriteDatabasesStorage { } } - internal func removeFavoriteWithoutSync(id: String) { + internal func removeFavoritesWithoutSync(ids: Set) { + guard !ids.isEmpty else { return } commit(sync: .discard) { favorites in - favorites = favorites.filter { Self.syncId(for: $0) != id } + favorites = favorites.filter { !ids.contains(Self.syncId(for: $0)) } } } @@ -116,6 +117,10 @@ internal final class FavoriteDatabasesStorage { case discard } + private static func syncIdsByConnection(of entries: Set) -> [UUID: Set] { + Dictionary(grouping: entries, by: \.connectionId).mapValues { Set($0.map(syncId(for:))) } + } + private static func upsert(_ entry: FavoriteDatabaseEntry, into favorites: inout Set) { guard !entry.database.isEmpty else { return } if let existing = favorites.first(where: { $0.id == entry.id }) { @@ -131,17 +136,17 @@ internal final class FavoriteDatabasesStorage { edit(&favorites) let previousById = Dictionary(previous.map { ($0.id, $0) }, uniquingKeysWith: { first, _ in first }) let currentIds = Set(favorites.map(\.id)) - let removedIds = previous.filter { !currentIds.contains($0.id) }.map(Self.syncId(for:)) + let removed = previous.filter { !currentIds.contains($0.id) } let changedIds = favorites.filter { previousById[$0.id] != $0 }.map(Self.syncId(for:)) - guard !removedIds.isEmpty || !changedIds.isEmpty else { return } + guard !removed.isEmpty || !changedIds.isEmpty else { return } persist(favorites) switch sync { case .track: - syncTracker.markDeleted(.favoriteDatabase, ids: removedIds) + syncTracker.markDeleted(.favoriteDatabase, idsByOwner: Self.syncIdsByConnection(of: removed)) syncTracker.markDirty(.favoriteDatabase, ids: changedIds) case .discard: - syncTracker.discardDirty(.favoriteDatabase, ids: removedIds) + syncTracker.discardDirty(.favoriteDatabase, ids: removed.map(Self.syncId(for:))) } postChangeNotification() } diff --git a/TablePro/Core/Storage/FavoriteTablesStorage.swift b/TablePro/Core/Storage/FavoriteTablesStorage.swift index fe5f9ddaf..b7ea4100f 100644 --- a/TablePro/Core/Storage/FavoriteTablesStorage.swift +++ b/TablePro/Core/Storage/FavoriteTablesStorage.swift @@ -156,7 +156,7 @@ final class FavoriteTablesStorage: @unchecked Sendable { @MainActor @discardableResult - func applyRemote(saved: [FavoriteEntry], deletedIds: Set) -> Set { + func applyRemote(saved: [FavoriteEntry], deletedIds: Set) -> [UUID: Set] { var removedThroughAliases: Set = [] let change = mutateState { favorites in favorites.formUnion(saved) @@ -166,7 +166,7 @@ final class FavoriteTablesStorage: @unchecked Sendable { if change.changesEntries { NotificationCenter.default.post(name: .favoriteTablesDidChange, object: self) } - return Set(removedThroughAliases.map(Self.syncId(for:))) + return Self.syncIdsByConnection(of: removedThroughAliases) } @MainActor @@ -222,14 +222,14 @@ final class FavoriteTablesStorage: @unchecked Sendable { remaining = favorites } guard change.changesEntries else { return change } - let removedIds = Self.recordIds(of: change.removed, keepingAliasesOf: remaining) + let removedIds = Self.recordIdsByConnection(of: change.removed, keepingAliasesOf: remaining) switch sync { case .track: let reclaimedAliases = Self.legacyAliases(of: change.removed).intersection(Self.legacyAliases(of: remaining)) - syncTracker.markDeleted(.tableFavorite, ids: Array(removedIds)) + syncTracker.markDeleted(.tableFavorite, idsByOwner: removedIds) syncTracker.markDirty(.tableFavorite, ids: Array(Self.recordIds(of: change.added).union(reclaimedAliases))) case .discard: - syncTracker.discardDirty(.tableFavorite, ids: Array(removedIds)) + syncTracker.discardDirty(.tableFavorite, ids: removedIds.values.flatMap { $0 }) } NotificationCenter.default.post(name: .favoriteTablesDidChange, object: self) return change @@ -253,13 +253,22 @@ final class FavoriteTablesStorage: @unchecked Sendable { Set(favorites.map(syncId(for:))).union(legacyAliases(of: favorites)) } - private static func recordIds( + private static func recordIdsByConnection( of removed: Set, keepingAliasesOf remaining: Set - ) -> Set { + ) -> [UUID: Set] { let claimedAliases = legacyAliases(of: remaining) - let retiredAliases = legacyAliases(of: removed).subtracting(claimedAliases) - return Set(removed.map(syncId(for:))).union(retiredAliases) + var ids: [UUID: Set] = [:] + for entry in removed { + ids[entry.connectionId, default: []].insert(syncId(for: entry)) + guard let alias = legacyAlias(of: entry), !claimedAliases.contains(alias) else { continue } + ids[entry.connectionId, default: []].insert(alias) + } + return ids + } + + private static func syncIdsByConnection(of entries: Set) -> [UUID: Set] { + Dictionary(grouping: entries, by: \.connectionId).mapValues { Set($0.map(syncId(for:))) } } private static func remove(_ deletedIds: Set, from favorites: inout Set) -> Set { diff --git a/TablePro/Core/Storage/SQLFavoriteManager.swift b/TablePro/Core/Storage/SQLFavoriteManager.swift index 85d91d4e0..2559d2123 100644 --- a/TablePro/Core/Storage/SQLFavoriteManager.swift +++ b/TablePro/Core/Storage/SQLFavoriteManager.swift @@ -45,8 +45,8 @@ internal final class SQLFavoriteManager: @unchecked Sendable { func deleteFavorite(id: UUID) async -> Bool { await operations.run { [self] in - guard await storage.deleteFavorite(id: id) else { return false } - syncTracker.markDeleted(.favorite, id: id.uuidString) + guard let connectionIds = await storage.deleteFavorites(ids: [id]) else { return false } + markDeleted(.favorite, ids: [id], connectionIds: connectionIds) postUpdateNotification(connectionId: nil) return true } @@ -54,10 +54,8 @@ internal final class SQLFavoriteManager: @unchecked Sendable { func deleteFavorites(ids: [UUID]) async { await operations.run { [self] in - guard await storage.deleteFavorites(ids: ids) else { return } - for id in ids { - syncTracker.markDeleted(.favorite, id: id.uuidString) - } + guard let connectionIds = await storage.deleteFavorites(ids: ids) else { return } + markDeleted(.favorite, ids: ids, connectionIds: connectionIds) postUpdateNotification(connectionId: nil) } } @@ -72,14 +70,10 @@ internal final class SQLFavoriteManager: @unchecked Sendable { /// holding the record re-uploads what was just deleted. func removeFavoritesAndFolders(for connectionId: UUID) async { await operations.run { [self] in - let removed = await storage.deleteFavoritesAndFolders(connectionId: connectionId) - guard !removed.isEmpty else { return } - for id in removed.favorites { - syncTracker.markDeleted(.favorite, id: id.uuidString) - } - for id in removed.folders { - syncTracker.markDeleted(.favoriteFolder, id: id.uuidString) - } + guard let removed = await storage.deleteFavoritesAndFolders(connectionId: connectionId), + !removed.isEmpty else { return } + syncTracker.markDeleted(.favorite, ids: removed.favorites.map(\.uuidString), owner: connectionId) + syncTracker.markDeleted(.favoriteFolder, ids: removed.folders.map(\.uuidString), owner: connectionId) markDetachedDirty(removed.detached) postUpdateNotification(connectionId: nil) } @@ -99,16 +93,26 @@ internal final class SQLFavoriteManager: @unchecked Sendable { syncTracker.markDirty(.favoriteFolder, ids: detached.folders.map(\.uuidString)) } + @MainActor + private func markDeleted(_ type: SyncRecordType, ids: [UUID], connectionIds: [UUID: UUID]) { + let idsByConnection = Dictionary(grouping: ids) { connectionIds[$0] } + for (connectionId, deletedIds) in idsByConnection { + syncTracker.markDeleted(type, ids: deletedIds.map(\.uuidString), owner: connectionId) + } + } + /// Used when another device deleted the connection. Marking tombstones here would push its own /// deletion straight back at it, which is the reason `FavoriteTablesStorage` splits the same /// way. - func removeFavoritesAndFoldersWithoutSync(for connectionId: UUID) async { + @discardableResult + func removeFavoritesAndFoldersWithoutSync(for connectionId: UUID) async -> Bool { await operations.run { [self] in - let removed = await storage.deleteFavoritesAndFolders(connectionId: connectionId) - guard !removed.isEmpty else { return } + guard let removed = await storage.deleteFavoritesAndFolders(connectionId: connectionId) else { return false } + guard !removed.isEmpty else { return true } syncTracker.discardDirty(.favorite, ids: removed.favorites.map(\.uuidString)) syncTracker.discardDirty(.favoriteFolder, ids: removed.folders.map(\.uuidString)) postUpdateNotification(connectionId: nil) + return true } } @@ -195,7 +199,7 @@ internal final class SQLFavoriteManager: @unchecked Sendable { func deleteFolder(id: UUID) async -> Bool { await operations.run { [self] in guard let deletion = await storage.deleteFolder(id: id) else { return false } - syncTracker.markDeleted(.favoriteFolder, id: id.uuidString) + syncTracker.markDeleted(.favoriteFolder, ids: [id.uuidString], owner: deletion.connectionId) syncTracker.markDirty(.favorite, ids: deletion.movedFavorites.map(\.uuidString)) syncTracker.markDirty(.favoriteFolder, ids: deletion.movedFolders.map(\.uuidString)) postUpdateNotification(connectionId: nil) @@ -286,7 +290,7 @@ internal final class SQLFavoriteManager: @unchecked Sendable { @MainActor private func applyRemoteFavoriteDeletions(_ ids: Set) async -> Bool { guard !ids.isEmpty else { return true } - guard await storage.deleteFavorites(ids: Array(ids)) else { return false } + guard await storage.deleteFavorites(ids: Array(ids)) != nil else { return false } syncTracker.discardDirty(.favorite, ids: ids.map(\.uuidString)) postUpdateNotification(connectionId: nil) return true diff --git a/TablePro/Core/Storage/SQLFavoriteStorage+Deletion.swift b/TablePro/Core/Storage/SQLFavoriteStorage+Deletion.swift new file mode 100644 index 000000000..26c0cfcb6 --- /dev/null +++ b/TablePro/Core/Storage/SQLFavoriteStorage+Deletion.swift @@ -0,0 +1,44 @@ +import Foundation +import SQLite3 + +internal extension SQLFavoriteStorage { + func deleteFavorites(ids: [UUID]) -> [UUID: UUID]? { + guard !ids.isEmpty else { return [:] } + let placeholders = ids.map { _ in "?" }.joined(separator: ",") + let bindings = ids.map(\.uuidString) + + guard let connectionIds = connectionIds( + of: "SELECT id, connection_id FROM favorites WHERE id IN (\(placeholders)) AND connection_id IS NOT NULL;", + bindings: bindings + ), run("DELETE FROM favorites WHERE id IN (\(placeholders));", bindings: bindings) else { + return nil + } + return connectionIds + } + + private func connectionIds(of sql: String, bindings: [String]) -> [UUID: UUID]? { + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return nil } + defer { sqlite3_finalize(statement) } + let transient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + for (index, value) in bindings.enumerated() { + sqlite3_bind_text(statement, Int32(index + 1), value, -1, transient) + } + + var connectionIds: [UUID: UUID] = [:] + while true { + switch sqlite3_step(statement) { + case SQLITE_ROW: + guard let rawId = sqlite3_column_text(statement, 0), + let rawConnectionId = sqlite3_column_text(statement, 1), + let id = UUID(uuidString: String(cString: rawId)), + let connectionId = UUID(uuidString: String(cString: rawConnectionId)) else { continue } + connectionIds[id] = connectionId + case SQLITE_DONE: + return connectionIds + default: + return nil + } + } + } +} diff --git a/TablePro/Core/Storage/SQLFavoriteStorage.swift b/TablePro/Core/Storage/SQLFavoriteStorage.swift index 227e61238..cb914a268 100644 --- a/TablePro/Core/Storage/SQLFavoriteStorage.swift +++ b/TablePro/Core/Storage/SQLFavoriteStorage.swift @@ -460,45 +460,6 @@ internal actor SQLFavoriteStorage { return result } - func deleteFavorite(id: UUID) -> Bool { - let sql = "DELETE FROM favorites WHERE id = ?;" - var statement: OpaquePointer? - guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { - return false - } - - defer { sqlite3_finalize(statement) } - - let SQLITE_TRANSIENT = unsafeBitCast(-1, to: sqlite3_destructor_type.self) - sqlite3_bind_text(statement, 1, id.uuidString, -1, SQLITE_TRANSIENT) - return sqlite3_step(statement) == SQLITE_DONE - } - - func deleteFavorites(ids: [UUID]) -> Bool { - guard !ids.isEmpty else { return true } - - let placeholders = ids.map { _ in "?" }.joined(separator: ",") - let sql = "DELETE FROM favorites WHERE id IN (\(placeholders));" - - var statement: OpaquePointer? - guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { - return false - } - - defer { sqlite3_finalize(statement) } - - let SQLITE_TRANSIENT = unsafeBitCast(-1, to: sqlite3_destructor_type.self) - for (index, id) in ids.enumerated() { - sqlite3_bind_text(statement, Int32(index + 1), id.uuidString, -1, SQLITE_TRANSIENT) - } - - let result = sqlite3_step(statement) - if result != SQLITE_DONE { - Self.logger.error("Failed to batch delete favorites: \(String(cString: sqlite3_errmsg(self.db)))") - } - return result == SQLITE_DONE - } - /// Both tables point at `folders` by id with no foreign key behind either column, so a delete /// that removes a folder leaves whatever named it holding an id nothing answers to. /// @@ -542,8 +503,8 @@ internal actor SQLFavoriteStorage { /// each record for sync and cannot ask afterwards: the rows are gone. Reporting a bare `Bool` /// is why a deleted connection's favorites and folders lived on in CloudKit and came back on a /// fresh install. - func deleteFavoritesAndFolders(connectionId: UUID) -> DeletedFavoriteRecords { - guard sqlite3_exec(db, "BEGIN IMMEDIATE;", nil, nil, nil) == SQLITE_OK else { return .none } + func deleteFavoritesAndFolders(connectionId: UUID) -> DeletedFavoriteRecords? { + guard sqlite3_exec(db, "BEGIN IMMEDIATE;", nil, nil, nil) == SQLITE_OK else { return nil } let id = connectionId.uuidString /// Read inside the same transaction as the delete, so nothing can be added between the two @@ -555,10 +516,10 @@ internal actor SQLFavoriteStorage { run("DELETE FROM folders WHERE connection_id = ?;", bindings: [id]), let detached = detachDanglingFolderReferences() else { sqlite3_exec(db, "ROLLBACK;", nil, nil, nil) - return .none + return nil } - guard sqlite3_exec(db, "COMMIT;", nil, nil, nil) == SQLITE_OK else { return .none } + guard sqlite3_exec(db, "COMMIT;", nil, nil, nil) == SQLITE_OK else { return nil } return DeletedFavoriteRecords(favorites: favorites, folders: folders, detached: detached) } @@ -602,7 +563,7 @@ internal actor SQLFavoriteStorage { return result } - private func run(_ sql: String, bindings: [String] = []) -> Bool { + func run(_ sql: String, bindings: [String] = []) -> Bool { var statement: OpaquePointer? guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { Self.logger.error("Failed to prepare statement: \(String(cString: sqlite3_errmsg(self.db)))") @@ -929,7 +890,7 @@ internal actor SQLFavoriteStorage { let SQLITE_TRANSIENT = unsafeBitCast(-1, to: sqlite3_destructor_type.self) - let findParentSQL = "SELECT parent_id FROM folders WHERE id = ?;" + let findParentSQL = "SELECT parent_id, connection_id FROM folders WHERE id = ?;" var findStatement: OpaquePointer? guard sqlite3_prepare_v2(db, findParentSQL, -1, &findStatement, nil) == SQLITE_OK else { sqlite3_exec(db, "ROLLBACK;", nil, nil, nil) @@ -939,8 +900,10 @@ internal actor SQLFavoriteStorage { sqlite3_bind_text(findStatement, 1, idString, -1, SQLITE_TRANSIENT) var parentId: String? + var connectionId: UUID? if sqlite3_step(findStatement) == SQLITE_ROW { parentId = sqlite3_column_text(findStatement, 0).map { String(cString: $0) } + connectionId = sqlite3_column_text(findStatement, 1).flatMap { UUID(uuidString: String(cString: $0)) } } sqlite3_finalize(findStatement) @@ -1003,7 +966,7 @@ internal actor SQLFavoriteStorage { } guard sqlite3_exec(db, "COMMIT;", nil, nil, nil) == SQLITE_OK else { return nil } - return FolderDeletion(movedFavorites: movedFavorites, movedFolders: movedFolders) + return FolderDeletion(connectionId: connectionId, movedFavorites: movedFavorites, movedFolders: movedFolders) } func fetchFolders(connectionId: UUID? = nil) -> [SQLFavoriteFolder] { @@ -1264,6 +1227,7 @@ enum FavoriteScopeRead: Equatable { /// What deleting one folder moved up to its parent, so the caller can mark those records dirty. struct FolderDeletion: Equatable { + let connectionId: UUID? let movedFavorites: [UUID] let movedFolders: [UUID] } @@ -1294,8 +1258,6 @@ struct DeletedFavoriteRecords { self.detached = detached } - static let none = DeletedFavoriteRecords(favorites: [], folders: []) - var isEmpty: Bool { favorites.isEmpty && folders.isEmpty && detached.isEmpty } diff --git a/TablePro/Core/Sync/Extensions/SyncCoordinator+PushCollection.swift b/TablePro/Core/Sync/Extensions/SyncCoordinator+PushCollection.swift index f35bef2c2..762b721d0 100644 --- a/TablePro/Core/Sync/Extensions/SyncCoordinator+PushCollection.swift +++ b/TablePro/Core/Sync/Extensions/SyncCoordinator+PushCollection.swift @@ -13,76 +13,106 @@ struct SyncPushBatch { } } +enum SyncPushDisposition { + case push(CKRecord) + case hold + case drop +} + private struct BuiltRecord { let id: String let record: CKRecord } +private struct CollectedRecords { + private(set) var built: [BuiltRecord] = [] + private(set) var heldIds: Set = [] + private var resolvedIds: Set = [] + + func hasResolved(_ id: String) -> Bool { + resolvedIds.contains(id) + } + + mutating func add(_ id: String, _ disposition: SyncPushDisposition) { + switch disposition { + case .push(let record): + built.append(BuiltRecord(id: id, record: record)) + case .hold: + heldIds.insert(id) + case .drop: + return + } + resolvedIds.insert(id) + } +} + extension SyncCoordinator { func collectPushBatch( snapshot: SyncEditSnapshot, - settings: SyncSettings, + boundary: SyncBoundary, zoneID: CKRecordZone.ID ) async -> SyncPushBatch { var batch = SyncPushBatch() - if settings.syncConnections { + if boundary.includes(.connection) { let storage = services.connectionStorage - await collectRecords(of: .connection, snapshot: snapshot, into: &batch, zoneID: zoneID) { + await collectRecords(of: .connection, snapshot: snapshot, boundary: boundary, into: &batch, zoneID: zoneID) { let connections = storage.loadConnections() return storage.lastLoadFailed ? nil : connections - } record: { (connection: DatabaseConnection) -> CKRecord? in - guard !connection.localOnly else { return nil } + } disposition: { (connection: DatabaseConnection) in + guard connection.participatesInSync else { return .drop } let recordID = SyncRecordMapper.recordID(type: .connection, id: connection.id.uuidString, in: zoneID) - return SyncRecordMapper.toCKRecord(connection, in: zoneID, base: recordCache.record(for: recordID)) + return .push(SyncRecordMapper.toCKRecord(connection, in: zoneID, base: recordCache.record(for: recordID))) } } - if settings.syncGroupsAndTags { + if boundary.includes(.group) { let groupStorage = services.groupStorage - await collectRecords(of: .group, snapshot: snapshot, into: &batch, zoneID: zoneID) { + await collectRecords(of: .group, snapshot: snapshot, boundary: boundary, into: &batch, zoneID: zoneID) { let groups = groupStorage.loadGroups() return groupStorage.storeIsUnreadable ? nil : groups - } record: { (group: ConnectionGroup) in SyncRecordMapper.toCKRecord(group, in: zoneID) } + } disposition: { (group: ConnectionGroup) in .push(SyncRecordMapper.toCKRecord(group, in: zoneID)) } + } + if boundary.includes(.tag) { let tagStorage = services.tagStorage - await collectRecords(of: .tag, snapshot: snapshot, into: &batch, zoneID: zoneID) { + await collectRecords(of: .tag, snapshot: snapshot, boundary: boundary, into: &batch, zoneID: zoneID) { let tags = tagStorage.loadTags() return tagStorage.storeIsUnreadable ? nil : tags - } record: { (tag: ConnectionTag) in SyncRecordMapper.toCKRecord(tag, in: zoneID) } + } disposition: { (tag: ConnectionTag) in .push(SyncRecordMapper.toCKRecord(tag, in: zoneID)) } } - if settings.syncSSHProfiles { + if boundary.includes(.sshProfile) { let storage = services.sshProfileStorage - await collectRecords(of: .sshProfile, snapshot: snapshot, into: &batch, zoneID: zoneID) { + await collectRecords(of: .sshProfile, snapshot: snapshot, boundary: boundary, into: &batch, zoneID: zoneID) { let profiles = storage.loadProfiles() return storage.lastLoadFailed ? nil : profiles - } record: { (profile: SSHProfile) in SyncRecordMapper.toCKRecord(profile, in: zoneID) } + } disposition: { (profile: SSHProfile) in .push(SyncRecordMapper.toCKRecord(profile, in: zoneID)) } } - if settings.syncCredentialProfiles { + if boundary.includes(.credentialProfile) { let storage = services.credentialProfileStorage - await collectRecords(of: .credentialProfile, snapshot: snapshot, into: &batch, zoneID: zoneID) { + await collectRecords( + of: .credentialProfile, snapshot: snapshot, boundary: boundary, into: &batch, zoneID: zoneID + ) { let profiles = storage.loadProfiles() return storage.lastLoadFailed ? nil : profiles - } record: { (profile: CredentialProfile) in SyncRecordMapper.toCKRecord(profile, in: zoneID) } + } disposition: { (profile: CredentialProfile) in .push(SyncRecordMapper.toCKRecord(profile, in: zoneID)) } } - if settings.syncSettings { - collectSettings(snapshot: snapshot, into: &batch, zoneID: zoneID) + if boundary.includes(.settings) { + collectSettings(snapshot: snapshot, boundary: boundary, into: &batch, zoneID: zoneID) } - if settings.syncTableFavorites { - collectTableFavorites(snapshot: snapshot, into: &batch, zoneID: zoneID) + if boundary.includes(.tableFavorite) { + collectTableFavorites(snapshot: snapshot, boundary: boundary, into: &batch, zoneID: zoneID) } - if settings.syncDatabaseFavorites { - collectDatabaseFavorites(snapshot: snapshot, into: &batch, zoneID: zoneID) + if boundary.includes(.favoriteDatabase) { + collectDatabaseFavorites(snapshot: snapshot, boundary: boundary, into: &batch, zoneID: zoneID) } - if settings.syncSQLFavorites { - await collectSQLFavorites(snapshot: snapshot, into: &batch, zoneID: zoneID) - } + await collectSQLFavorites(snapshot: snapshot, boundary: boundary, into: &batch, zoneID: zoneID) return batch } @@ -90,55 +120,54 @@ extension SyncCoordinator { private func collectRecords( of type: SyncRecordType, snapshot: SyncEditSnapshot, + boundary: SyncBoundary, into batch: inout SyncPushBatch, zoneID: CKRecordZone.ID, loaded: () async -> [Record]?, - record: (Record) -> CKRecord? + disposition: (Record) -> SyncPushDisposition ) async where Record.ID == UUID { let dirtyIds = snapshot.dirtyIds(for: type) guard !dirtyIds.isEmpty else { - append(type, dirtyIds: dirtyIds, built: [], into: &batch, zoneID: zoneID) + append(type, dirtyIds: dirtyIds, collected: CollectedRecords(), boundary: boundary, into: &batch, zoneID: zoneID) return } - let built = await loaded().map { records in - Self.build(records, dirtyIds: dirtyIds, id: { $0.id.uuidString }, record: record) + let collected = await loaded().map { records in + Self.collect(records, dirtyIds: dirtyIds, id: { $0.id.uuidString }, disposition: disposition) } - append(type, dirtyIds: dirtyIds, built: built, into: &batch, zoneID: zoneID) + append(type, dirtyIds: dirtyIds, collected: collected, boundary: boundary, into: &batch, zoneID: zoneID) } - private static func build( + private static func collect( _ items: [Item], dirtyIds: Set, id: (Item) -> String, - record: (Item) -> CKRecord? - ) -> [BuiltRecord] { - var built: [BuiltRecord] = [] - var builtIds: Set = [] + disposition: (Item) -> SyncPushDisposition + ) -> CollectedRecords { + var collected = CollectedRecords() for item in items { let itemId = id(item) - guard dirtyIds.contains(itemId), !builtIds.contains(itemId), let record = record(item) else { continue } - builtIds.insert(itemId) - built.append(BuiltRecord(id: itemId, record: record)) + guard dirtyIds.contains(itemId), !collected.hasResolved(itemId) else { continue } + collected.add(itemId, disposition(item)) } - return built + return collected } private func append( _ type: SyncRecordType, dirtyIds: Set, - built: [BuiltRecord]?, - retaining retainedIds: Set = [], + collected: CollectedRecords?, + boundary: SyncBoundary, into batch: inout SyncPushBatch, zoneID: CKRecordZone.ID ) { - guard let built else { - appendTombstones(of: type, sparing: dirtyIds, to: &batch, zoneID: zoneID) + guard let collected else { + appendTombstones(of: type, sparing: dirtyIds, boundary: boundary, to: &batch, zoneID: zoneID) return } - let pushable = Set(built.map(\.id)) - batch.records.append(contentsOf: built.map(\.record)) - discardUnpushable(type, dirtyIds: dirtyIds.subtracting(retainedIds), pushable: pushable) - let superseded = appendTombstones(of: type, sparing: pushable, to: &batch, zoneID: zoneID) + let pushable = Set(collected.built.map(\.id)) + batch.records.append(contentsOf: collected.built.map(\.record)) + discardUnpushable(type, dirtyIds: dirtyIds.subtracting(collected.heldIds), pushable: pushable) + let superseded = appendTombstones(of: type, sparing: pushable, boundary: boundary, to: &batch, zoneID: zoneID) batch.supersededTombstones.formUnion(superseded.map { SyncRecordIdentity(type: type, id: $0) }) } @@ -155,115 +184,136 @@ extension SyncCoordinator { private func appendTombstones( of type: SyncRecordType, sparing sparedIds: Set, + boundary: SyncBoundary, to batch: inout SyncPushBatch, zoneID: CKRecordZone.ID ) -> Set { var spared: Set = [] + var heldCount = 0 for tombstone in metadataStorage.tombstones(for: type) { guard !sparedIds.contains(tombstone.id) else { spared.insert(tombstone.id) continue } + guard boundary.includes(tombstone, of: type) else { + heldCount += 1 + continue + } batch.deletions.append(SyncRecordMapper.recordID(type: type, id: tombstone.id, in: zoneID)) } + if heldCount > 0 { + Self.logger.info( + "Held \(heldCount, privacy: .public) \(type.rawValue, privacy: .public) deletions of connections this Mac does not sync" + ) + } return spared } - private func collectSettings(snapshot: SyncEditSnapshot, into batch: inout SyncPushBatch, zoneID: CKRecordZone.ID) { - for category in snapshot.dirtyIds(for: .settings) { - guard let data = settingsData(for: category) else { continue } - batch.records.append(SyncRecordMapper.toCKRecord(category: category, settingsData: data, in: zoneID)) + private func collectSettings( + snapshot: SyncEditSnapshot, + boundary: SyncBoundary, + into batch: inout SyncPushBatch, + zoneID: CKRecordZone.ID + ) { + let dirtyIds = snapshot.dirtyIds(for: .settings) + retireSettingsTombstonesOfLiveCategories(dirtyIds: dirtyIds) + let collected = Self.collect(Array(dirtyIds), dirtyIds: dirtyIds, id: { $0 }) { category in + let owner = SyncBoundary.owner(ofRecordId: category, type: .settings) + guard boundary.includes(.settings, owner: owner) else { return .hold } + guard let data = settingsData(for: category) else { return .drop } + return .push(SyncRecordMapper.toCKRecord(category: category, settingsData: data, in: zoneID)) + } + append(.settings, dirtyIds: dirtyIds, collected: collected, boundary: boundary, into: &batch, zoneID: zoneID) + } + + private func retireSettingsTombstonesOfLiveCategories(dirtyIds: Set) { + let live = metadataStorage.tombstones(for: .settings) + .map(\.id) + .filter { !dirtyIds.contains($0) && settingsData(for: $0) != nil } + for category in Set(live) { + metadataStorage.removeTombstone(category, type: .settings) } } private func collectTableFavorites( snapshot: SyncEditSnapshot, + boundary: SyncBoundary, into batch: inout SyncPushBatch, zoneID: CKRecordZone.ID ) { let dirtyIds = snapshot.dirtyIds(for: .tableFavorite) - guard !dirtyIds.isEmpty else { - append(.tableFavorite, dirtyIds: dirtyIds, built: [], into: &batch, zoneID: zoneID) - return - } - let built = Self.buildTableFavorites( - services.favoriteTablesStorage.loadFavorites(), - dirtyIds: dirtyIds, - zoneID: zoneID - ) - append(.tableFavorite, dirtyIds: dirtyIds, built: built, into: &batch, zoneID: zoneID) + let favorites = dirtyIds.isEmpty ? [] : services.favoriteTablesStorage.loadFavorites() + let collected = Self.collectTableFavorites(favorites, dirtyIds: dirtyIds, boundary: boundary, zoneID: zoneID) + append(.tableFavorite, dirtyIds: dirtyIds, collected: collected, boundary: boundary, into: &batch, zoneID: zoneID) } - private static func buildTableFavorites( + private static func collectTableFavorites( _ favorites: Set, dirtyIds: Set, + boundary: SyncBoundary, zoneID: CKRecordZone.ID - ) -> [BuiltRecord] { + ) -> CollectedRecords { let claims = FavoriteTablesStorage.aliasClaims(in: favorites) - var built: [BuiltRecord] = [] + var collected = CollectedRecords() for entry in favorites { - let currentId = FavoriteTablesStorage.syncId(for: entry) - if dirtyIds.contains(currentId) { - built.append(BuiltRecord( - id: currentId, - record: SyncRecordMapper.toCKRecord(favoriteEntry: entry, recordId: currentId, in: zoneID) - )) + var recordIds = [FavoriteTablesStorage.syncId(for: entry)] + if let alias = FavoriteTablesStorage.legacyAlias(of: entry), claims[alias]?.count == 1 { + recordIds.append(alias) + } + let admitted = boundary.includes(.tableFavorite, owner: entry.connectionId) + for recordId in recordIds where dirtyIds.contains(recordId) { + collected.add(recordId, admitted ? .push( + SyncRecordMapper.toCKRecord(favoriteEntry: entry, recordId: recordId, in: zoneID) + ) : .hold) } - guard let alias = FavoriteTablesStorage.legacyAlias(of: entry), - dirtyIds.contains(alias), - claims[alias]?.count == 1 else { continue } - built.append(BuiltRecord( - id: alias, - record: SyncRecordMapper.toCKRecord(favoriteEntry: entry, recordId: alias, in: zoneID) - )) } - return built + return collected } - /// A connection the user marked local only never reaches iCloud, and neither do the database - /// names hanging off it. Tombstones are not filtered: a deletion only ever removes something, - /// and a connection can be marked local only after its favorites were already pushed. private func collectDatabaseFavorites( snapshot: SyncEditSnapshot, + boundary: SyncBoundary, into batch: inout SyncPushBatch, zoneID: CKRecordZone.ID ) { let dirtyIds = snapshot.dirtyIds(for: .favoriteDatabase) - guard !dirtyIds.isEmpty else { - append(.favoriteDatabase, dirtyIds: dirtyIds, built: [], into: &batch, zoneID: zoneID) - return - } - let localOnlyIds = Set(services.connectionStorage.loadConnections().filter(\.localOnly).map(\.id)) - let favorites = services.favoriteDatabasesStorage.loadFavorites() - let withheld = favorites.filter { localOnlyIds.contains($0.connectionId) } - let built = Self.build( - Array(favorites.subtracting(withheld)), - dirtyIds: dirtyIds, - id: FavoriteDatabasesStorage.syncId(for:), - record: { SyncRecordMapper.toCKRecord(favoriteDatabase: $0, in: zoneID) } - ) - append( - .favoriteDatabase, + let favorites = dirtyIds.isEmpty ? [] : services.favoriteDatabasesStorage.loadFavorites() + let collected = Self.collect( + Array(favorites), dirtyIds: dirtyIds, - built: built, - retaining: Set(withheld.map(FavoriteDatabasesStorage.syncId(for:))), - into: &batch, - zoneID: zoneID - ) + id: FavoriteDatabasesStorage.syncId(for:) + ) { entry in + guard boundary.includes(.favoriteDatabase, owner: entry.connectionId) else { return .hold } + return .push(SyncRecordMapper.toCKRecord(favoriteDatabase: entry, in: zoneID)) + } + append(.favoriteDatabase, dirtyIds: dirtyIds, collected: collected, boundary: boundary, into: &batch, zoneID: zoneID) } private func collectSQLFavorites( snapshot: SyncEditSnapshot, + boundary: SyncBoundary, into batch: inout SyncPushBatch, zoneID: CKRecordZone.ID ) async { let manager = services.sqlFavoriteManager - await collectRecords(of: .favorite, snapshot: snapshot, into: &batch, zoneID: zoneID) { - await manager.favoritesForSync() - } record: { (favorite: SQLFavorite) in SyncRecordMapper.toCKRecord(sqlFavorite: favorite, in: zoneID) } + if boundary.includes(.favorite) { + await collectRecords(of: .favorite, snapshot: snapshot, boundary: boundary, into: &batch, zoneID: zoneID) { + await manager.favoritesForSync() + } disposition: { (favorite: SQLFavorite) in + guard boundary.includes(.favorite, owner: favorite.connectionId) else { return .hold } + return .push(SyncRecordMapper.toCKRecord(sqlFavorite: favorite, in: zoneID)) + } + } - await collectRecords(of: .favoriteFolder, snapshot: snapshot, into: &batch, zoneID: zoneID) { - await manager.foldersForSync() - } record: { (folder: SQLFavoriteFolder) in SyncRecordMapper.toCKRecord(sqlFavoriteFolder: folder, in: zoneID) } + if boundary.includes(.favoriteFolder) { + await collectRecords( + of: .favoriteFolder, snapshot: snapshot, boundary: boundary, into: &batch, zoneID: zoneID + ) { + await manager.foldersForSync() + } disposition: { (folder: SQLFavoriteFolder) in + guard boundary.includes(.favoriteFolder, owner: folder.connectionId) else { return .hold } + return .push(SyncRecordMapper.toCKRecord(sqlFavoriteFolder: folder, in: zoneID)) + } + } } } diff --git a/TablePro/Core/Sync/Extensions/SyncCoordinator+RemoteDeletions.swift b/TablePro/Core/Sync/Extensions/SyncCoordinator+RemoteDeletions.swift index a3b12d043..b402d81e0 100644 --- a/TablePro/Core/Sync/Extensions/SyncCoordinator+RemoteDeletions.swift +++ b/TablePro/Core/Sync/Extensions/SyncCoordinator+RemoteDeletions.swift @@ -10,6 +10,8 @@ struct SyncPendingDeletions: Equatable { var sshProfiles: Set = [] var credentialProfiles: Set = [] var tableFavorites: Set = [] + var databaseFavorites: Set = [] + var settingsRecordNames: Set = [] var sqlFavorites: Set = [] var sqlFolders: Set = [] @@ -38,8 +40,10 @@ struct SyncPendingDeletions: Equatable { if let uuid { credentialProfiles.insert(uuid) } case .tableFavorite: tableFavorites.insert(id) - case .favoriteDatabase, .settings: - return + case .favoriteDatabase: + databaseFavorites.insert(id) + case .settings: + settingsRecordNames.insert(type.recordNamePrefix + id) case .favorite: if let uuid { sqlFavorites.insert(uuid) } case .favoriteFolder: @@ -52,7 +56,7 @@ struct SyncRemoteDeletionEffects { var connectionsChanged = false var groupsOrTagsChanged = false var persistenceFailed = false - var tableFavoriteIdsToRetire: Set = [] + var tableFavoriteIdsToRetire: [UUID: Set] = [:] } extension SyncCoordinator { @@ -67,13 +71,15 @@ extension SyncCoordinator { saved: tableFavorites, deletedIds: pending.tableFavorites ) + services.favoriteDatabasesStorage.removeFavoritesWithoutSync(ids: pending.databaseFavorites) let persisted = [ applyRemoteConnectionDeletions(pending.connections), applyRemoteGroupDeletions(pending.groups), applyRemoteTagDeletions(pending.tags), applyRemoteSSHProfileDeletions(pending.sshProfiles), - applyRemoteCredentialProfileDeletions(pending.credentialProfiles) + applyRemoteCredentialProfileDeletions(pending.credentialProfiles), + applyRemoteColumnLayoutDeletions(pending.settingsRecordNames) ] effects.persistenceFailed = persisted.contains(false) return effects @@ -101,6 +107,12 @@ extension SyncCoordinator { return true } + private func applyRemoteColumnLayoutDeletions(_ recordNames: Set) -> Bool { + guard !recordNames.isEmpty else { return true } + let persister = columnLayouts() + return persister.removeWithoutSync(storageKeys: persister.storageKeys(forSyncRecordNames: recordNames)) + } + private func applyRemoteGroupDeletions(_ ids: Set) -> Bool { guard !ids.isEmpty else { return true } var groups = services.groupStorage.loadGroups() diff --git a/TablePro/Core/Sync/Extensions/SyncCoordinator+Scope.swift b/TablePro/Core/Sync/Extensions/SyncCoordinator+Scope.swift new file mode 100644 index 000000000..0a9754564 --- /dev/null +++ b/TablePro/Core/Sync/Extensions/SyncCoordinator+Scope.swift @@ -0,0 +1,22 @@ +import Foundation +import TableProSyncTransport + +extension SyncCoordinator { + func syncBoundary(settings: SyncSettings) -> SyncBoundary { + let storage = services.connectionStorage + let connections = storage.loadConnections() + return SyncBoundary( + settings: settings, + connections: storage.lastLoadFailed ? nil : connections, + ownersKeptOffSync: changeTracker.ownersKeptOffSync + ) + } + + func pruneTombstones(within boundary: SyncBoundary) { + metadataStorage.pruneTombstones(olderThan: Self.tombstoneRetentionDays) { type, tombstone in + boundary.includes(tombstone, of: type) + } + } + + private static let tombstoneRetentionDays = 30 +} diff --git a/TablePro/Core/Sync/SyncBoundary.swift b/TablePro/Core/Sync/SyncBoundary.swift new file mode 100644 index 000000000..7047f282d --- /dev/null +++ b/TablePro/Core/Sync/SyncBoundary.swift @@ -0,0 +1,60 @@ +import Foundation +import TableProSyncTransport + +struct SyncBoundary: Equatable, Sendable { + let includedTypes: Set + let excludedConnectionIds: Set? + + init(includedTypes: Set, excludedConnectionIds: Set?) { + self.includedTypes = includedTypes + self.excludedConnectionIds = excludedConnectionIds + } + + init( + settings: SyncSettings, + connections: [DatabaseConnection]?, + ownersKeptOffSync: Set = [], + writableTypes: Set = SyncRecordType.verifiedInProduction + ) { + self.init( + includedTypes: Set(SyncRecordType.allCases.filter { settings.syncs($0) && writableTypes.contains($0) }), + excludedConnectionIds: connections.map { connections in + let keptOff = connections.filter { !$0.participatesInSync }.map(\.id) + return Set(keptOff).union(ownersKeptOffSync.subtracting(connections.map(\.id))) + } + ) + } + + var knowsOwners: Bool { + excludedConnectionIds != nil + } + + func includes(_ type: SyncRecordType) -> Bool { + includedTypes.contains(type) + } + + func includes(_ type: SyncRecordType, owner: UUID?) -> Bool { + guard includes(type) else { return false } + guard let owner else { return true } + guard let excludedConnectionIds else { return false } + return !excludedConnectionIds.contains(owner) + } + + func includes(_ tombstone: Tombstone, of type: SyncRecordType) -> Bool { + includes(type, owner: Self.owner(of: tombstone, type: type)) + } + + static func owner(of tombstone: Tombstone, type: SyncRecordType) -> UUID? { + tombstone.owner ?? owner(ofRecordId: tombstone.id, type: type) + } + + static func owner(ofRecordId id: String, type: SyncRecordType) -> UUID? { + switch type { + case .settings: + return FileColumnLayoutPersister.connectionId(ofSyncCategory: id) + case .connection, .group, .tag, .sshProfile, .credentialProfile, + .tableFavorite, .favoriteDatabase, .favorite, .favoriteFolder: + return nil + } + } +} diff --git a/TablePro/Core/Sync/SyncChangeTracker.swift b/TablePro/Core/Sync/SyncChangeTracker.swift index 2faea5c48..cd06aedaa 100644 --- a/TablePro/Core/Sync/SyncChangeTracker.swift +++ b/TablePro/Core/Sync/SyncChangeTracker.swift @@ -81,14 +81,55 @@ final class SyncChangeTracker: Sendable { @MainActor func markDeleted(_ type: SyncRecordType, ids: [String]) { + markDeleted(type, ids: ids, owner: nil) + } + + @MainActor + func markDeleted(_ type: SyncRecordType, ids: [String], owner: UUID?) { + guard !isSuppressed, !ids.isEmpty else { return } + metadataStorage.removeDirty(ids, type: type) + metadataStorage.addTombstones(ids, type: type, owner: owner) + recordEdits(type, ids: ids) + Self.logger.trace("Marked deleted: \(type.rawValue) x\(ids.count)") + postChangeNotification() + } + + @MainActor + func markDeleted(_ type: SyncRecordType, idsByOwner: [UUID: Set]) { + let ids = idsByOwner.values.flatMap { $0 } guard !isSuppressed, !ids.isEmpty else { return } metadataStorage.removeDirty(ids, type: type) - metadataStorage.addTombstones(ids, type: type) + metadataStorage.addTombstones( + idsByOwner.flatMap { owner, ownedIds in ownedIds.map { Tombstone(id: $0, owner: owner) } }, + type: type + ) recordEdits(type, ids: ids) Self.logger.trace("Marked deleted: \(type.rawValue) x\(ids.count)") postChangeNotification() } + @MainActor + func discardTombstones(ownedBy owners: Set) { + guard !owners.isEmpty else { return } + metadataStorage.removeTombstones { type, tombstone in + SyncBoundary.owner(of: tombstone, type: type).map(owners.contains) ?? false + } + } + + var ownersKeptOffSync: Set { + metadataStorage.ownersKeptOffSync() + } + + @MainActor + func keepOffSync(owners: Set) { + metadataStorage.keepOffSync(owners: owners) + } + + @MainActor + func releaseOwnersKeptOffSync(_ owners: Set) { + metadataStorage.releaseOwnersKeptOffSync(owners) + } + // MARK: - Query func dirtyRecords(for type: SyncRecordType) -> Set { diff --git a/TablePro/Core/Sync/SyncCoordinator.swift b/TablePro/Core/Sync/SyncCoordinator.swift index 552a80e13..06f53533b 100644 --- a/TablePro/Core/Sync/SyncCoordinator.swift +++ b/TablePro/Core/Sync/SyncCoordinator.swift @@ -127,7 +127,6 @@ final class SyncCoordinator: ObservableObject { lastSyncDate = Date() metadataStorage.lastSyncDate = lastSyncDate settle(.idle, from: generation) - metadataStorage.pruneTombstones(olderThan: 30) Self.logger.info("Sync completed successfully") } @@ -249,7 +248,7 @@ final class SyncCoordinator: ObservableObject { let connections = services.connectionStorage.loadConnections() changeTracker.markDirty( .connection, - ids: connections.filter { !$0.localOnly }.map { $0.id.uuidString } + ids: connections.filter(\.participatesInSync).map { $0.id.uuidString } ) let groups = services.groupStorage.loadGroups() @@ -382,12 +381,15 @@ final class SyncCoordinator: ObservableObject { private func performPush() async -> PushReport { let snapshot = changeTracker.editSnapshot() - let settings = services.appSettingsStorage.loadSync() + let boundary = syncBoundary(settings: services.appSettingsStorage.loadSync()) let zoneID = await transport.currentZoneID - let batch = await collectPushBatch(snapshot: snapshot, settings: settings, zoneID: zoneID) + let batch = await collectPushBatch(snapshot: snapshot, boundary: boundary, zoneID: zoneID) let deletions = batch.uniqueDeletions - guard !batch.records.isEmpty || !deletions.isEmpty else { return PushReport() } + guard !batch.records.isEmpty || !deletions.isEmpty else { + pruneTombstones(within: boundary) + return PushReport() + } let identities = SyncRecordMapper.identities(for: pushedLocalIds(snapshot), in: zoneID) var outcome: PushOutcome @@ -426,6 +428,7 @@ final class SyncCoordinator: ObservableObject { return PushReport(echoGuard: echoGuard, error: interruption) } guard outcome.hasFailures, let firstFailure = outcome.failures.values.first else { + pruneTombstones(within: boundary) return PushReport(echoGuard: echoGuard) } let rejection = SyncError.pushRejected(count: outcome.failures.count, detail: firstFailure.message) @@ -558,7 +561,7 @@ final class SyncCoordinator: ObservableObject { ) changeTracker.isSuppressed = false - changeTracker.markDeleted(.tableFavorite, ids: Array(effects.tableFavoriteIdsToRetire)) + changeTracker.markDeleted(.tableFavorite, idsByOwner: effects.tableFavoriteIdsToRetire) return !effects.persistenceFailed } @@ -577,6 +580,7 @@ final class SyncCoordinator: ObservableObject { let tagTombstoneIds = Set(metadataStorage.tombstones(for: .tag).map(\.id)) let sshTombstoneIds = Set(metadataStorage.tombstones(for: .sshProfile).map(\.id)) let credentialTombstoneIds = Set(metadataStorage.tombstones(for: .credentialProfile).map(\.id)) + let settingsTombstoneIds = Set(metadataStorage.tombstones(for: .settings).map(\.id)) let tableFavoriteTombstoneIds = Set(metadataStorage.tombstones(for: .tableFavorite).map(\.id)) var tableFavorites: [FavoriteTablesStorage.FavoriteEntry] = [] let databaseFavoriteTombstoneIds = Set(metadataStorage.tombstones(for: .favoriteDatabase).map(\.id)) @@ -613,7 +617,7 @@ final class SyncCoordinator: ObservableObject { persistenceFailed = true } case .settings: - applyRemoteSettings(record) + applyRemoteSettings(record, tombstoneIds: settingsTombstoneIds) case .tableFavorite: if let favorite = remoteTableFavorite(record, tombstoneIds: tableFavoriteTombstoneIds) { tableFavorites.append(favorite) @@ -824,8 +828,9 @@ final class SyncCoordinator: ObservableObject { services.sshProfileStorage.refreshLinkedConnections(with: remoteProfile) } - private func applyRemoteSettings(_ record: CKRecord) { + private func applyRemoteSettings(_ record: CKRecord, tombstoneIds: Set) { guard let category = SyncRecordMapper.settingsCategory(from: record), + !tombstoneIds.contains(category), let data = SyncRecordMapper.settingsData(from: record) else { return } do { diff --git a/TablePro/Models/Connection/DatabaseConnection.swift b/TablePro/Models/Connection/DatabaseConnection.swift index bced2cce1..10ca1821f 100644 --- a/TablePro/Models/Connection/DatabaseConnection.swift +++ b/TablePro/Models/Connection/DatabaseConnection.swift @@ -383,6 +383,10 @@ extension DatabaseConnection { // MARK: - Device-Local State internal extension DatabaseConnection { + var participatesInSync: Bool { + !localOnly && !isSample + } + func adoptingDeviceLocalState(from local: DatabaseConnection) -> DatabaseConnection { var adopted = self adopted.localOnly = local.localOnly diff --git a/TableProTests/Core/Storage/FavoriteDatabasesStorageTests.swift b/TableProTests/Core/Storage/FavoriteDatabasesStorageTests.swift index 6a471f635..ef021d007 100644 --- a/TableProTests/Core/Storage/FavoriteDatabasesStorageTests.swift +++ b/TableProTests/Core/Storage/FavoriteDatabasesStorageTests.swift @@ -4,8 +4,8 @@ // import Foundation -import Testing import TableProSyncTransport +import Testing @testable import TablePro @@ -230,7 +230,7 @@ struct FavoriteDatabasesStorageTests { storage.setFavoriteWithoutSync(entry) #expect(storage.favorites(for: connectionId).first?.environment == .testing) - storage.removeFavoriteWithoutSync(id: FavoriteDatabasesStorage.syncId(for: entry)) + storage.removeFavoritesWithoutSync(ids: [FavoriteDatabasesStorage.syncId(for: entry)]) #expect(storage.favorites(for: connectionId).isEmpty) #expect(metadata.dirtyIds(for: .favoriteDatabase).isEmpty) #expect(metadata.tombstones(for: .favoriteDatabase).isEmpty) diff --git a/TableProTests/Core/Storage/SQLFavoriteStorageTests.swift b/TableProTests/Core/Storage/SQLFavoriteStorageTests.swift index 7c3a14094..a0ff23b1f 100644 --- a/TableProTests/Core/Storage/SQLFavoriteStorageTests.swift +++ b/TableProTests/Core/Storage/SQLFavoriteStorageTests.swift @@ -89,7 +89,7 @@ struct SQLFavoriteStorageTests { let fav = makeFavorite() _ = await storage.addFavorite(fav) - let deleted = await storage.deleteFavorite(id: fav.id) + let deleted = await storage.deleteFavorites(ids: [fav.id]) != nil #expect(deleted) let fetched = await storage.fetchFavorites() diff --git a/TableProTests/Core/Storage/SQLFavoriteVersionTests.swift b/TableProTests/Core/Storage/SQLFavoriteVersionTests.swift index de726b4e6..b7b48aa88 100644 --- a/TableProTests/Core/Storage/SQLFavoriteVersionTests.swift +++ b/TableProTests/Core/Storage/SQLFavoriteVersionTests.swift @@ -104,7 +104,7 @@ struct SQLFavoriteVersionTests { #expect(await storage.updateFavorite(favorite).succeeded) #expect(await storage.fetchVersions(favoriteId: favorite.id).count == 1) - #expect(await storage.deleteFavorite(id: favorite.id)) + #expect(await storage.deleteFavorites(ids: [favorite.id]) != nil) #expect(await storage.fetchVersions(favoriteId: favorite.id).isEmpty) } @@ -135,7 +135,7 @@ struct SQLFavoriteVersionTests { favorite.query = "SELECT new" #expect(await storage.updateFavorite(favorite).succeeded) let old = try #require(await manager.fetchVersions(favoriteId: favorite.id).first) - #expect(await storage.deleteFavorite(id: favorite.id)) + #expect(await storage.deleteFavorites(ids: [favorite.id]) != nil) #expect(await manager.restore(old) == false) } diff --git a/TableProTests/Core/Sync/SyncBoundaryTests.swift b/TableProTests/Core/Sync/SyncBoundaryTests.swift new file mode 100644 index 000000000..4a6e5e1e1 --- /dev/null +++ b/TableProTests/Core/Sync/SyncBoundaryTests.swift @@ -0,0 +1,111 @@ +import Foundation +@testable import TablePro +import TableProSyncTransport +import Testing + +@MainActor +struct SyncBoundaryTests { + private static let everyCategoryOn = SyncSettings( + enabled: true, + syncConnections: true, + syncGroupsAndTags: true, + syncSettings: true + ) + + private static func connection(localOnly: Bool = false, isSample: Bool = false) -> DatabaseConnection { + var connection = TestFixtures.makeConnection() + connection.localOnly = localOnly + connection.isSample = isSample + return connection + } + + @Test("Every record type is in scope exactly when its category syncs and its type is deployed") + func typeScopeFollowsCategoryAndDeployment() { + var settings = Self.everyCategoryOn + settings.syncTableFavorites = false + let writable = SyncRecordType.verifiedInProduction.subtracting([.favoriteDatabase]) + + let boundary = SyncBoundary(settings: settings, connections: [], writableTypes: writable) + + for type in SyncRecordType.allCases { + let expected = settings.syncs(type) && writable.contains(type) + #expect(boundary.includes(type) == expected, "\(type.rawValue)") + } + #expect(!boundary.includes(.tableFavorite)) + #expect(!boundary.includes(.favoriteDatabase)) + #expect(boundary.includes(.favorite)) + } + + @Test("Local only and sample connections are refused as owners, synced and unknown ones are not") + func ownersKeptOffICloudAreRefused() { + let localOnly = Self.connection(localOnly: true) + let sample = Self.connection(isSample: true) + let synced = Self.connection() + let boundary = SyncBoundary(settings: Self.everyCategoryOn, connections: [localOnly, sample, synced]) + + #expect(boundary.excludedConnectionIds == [localOnly.id, sample.id]) + #expect(!boundary.includes(.tableFavorite, owner: localOnly.id)) + #expect(!boundary.includes(.favorite, owner: sample.id)) + #expect(boundary.includes(.tableFavorite, owner: synced.id)) + #expect(boundary.includes(.favoriteDatabase, owner: UUID())) + #expect(boundary.includes(.favorite, owner: nil)) + } + + @Test("A deleted connection still being purged stays refused, and a connection that exists again follows its own flag") + func ownersKeptOffSyncApplyOnlyToDeletedConnections() { + let deleted = UUID() + let reimported = Self.connection() + let boundary = SyncBoundary( + settings: Self.everyCategoryOn, + connections: [reimported], + ownersKeptOffSync: [deleted, reimported.id] + ) + + #expect(!boundary.includes(.favorite, owner: deleted)) + #expect(boundary.includes(.favorite, owner: reimported.id)) + } + + @Test("An owner in scope is still refused when its record type is not") + func ownerDoesNotOverrideCategory() { + var settings = Self.everyCategoryOn + settings.syncSQLFavorites = false + let boundary = SyncBoundary(settings: settings, connections: [Self.connection()]) + + #expect(!boundary.includes(.favorite, owner: nil)) + #expect(!boundary.includes(.favoriteFolder, owner: UUID())) + } + + @Test("A connection store that cannot be read holds every owned record and lets unowned ones through") + func unreadableStoreHoldsOwnedRecords() { + let boundary = SyncBoundary(settings: Self.everyCategoryOn, connections: nil) + + #expect(!boundary.knowsOwners) + #expect(!boundary.includes(.tableFavorite, owner: UUID())) + #expect(boundary.includes(.favorite, owner: nil)) + #expect(boundary.includes(.tag)) + } + + @Test("A tombstone is held by the owner it was written with") + func tombstoneHeldByItsOwner() { + let localOnly = Self.connection(localOnly: true) + let boundary = SyncBoundary(settings: Self.everyCategoryOn, connections: [localOnly]) + + #expect(!boundary.includes(Tombstone(id: "a", owner: localOnly.id), of: .tableFavorite)) + #expect(boundary.includes(Tombstone(id: "b", owner: UUID()), of: .tableFavorite)) + #expect(boundary.includes(Tombstone(id: "c"), of: .tableFavorite)) + } + + @Test("A column layout tombstone written before owners existed is held by the connection its name carries") + func legacyLayoutTombstoneOwnerComesFromItsName() { + let localOnly = Self.connection(localOnly: true) + let boundary = SyncBoundary(settings: Self.everyCategoryOn, connections: [localOnly]) + let key = ColumnLayoutTableKey( + connectionId: localOnly.id, databaseName: "shop", schemaName: "public", tableName: "orders" + ) + let category = FileColumnLayoutPersister.syncCategory(for: key.storageKey) + + #expect(SyncBoundary.owner(ofRecordId: category, type: .settings) == localOnly.id) + #expect(SyncBoundary.owner(ofRecordId: AppSettingsCategory.editor, type: .settings) == nil) + #expect(!boundary.includes(Tombstone(id: category), of: .settings)) + } +} diff --git a/TableProTests/Core/Sync/SyncLocalOnlyDependentsTests.swift b/TableProTests/Core/Sync/SyncLocalOnlyDependentsTests.swift new file mode 100644 index 000000000..96b90ecd0 --- /dev/null +++ b/TableProTests/Core/Sync/SyncLocalOnlyDependentsTests.swift @@ -0,0 +1,594 @@ +import CloudKit +import Foundation +@testable import TablePro +import TableProSyncTransport +import Testing + +@MainActor +struct SyncLocalOnlyDependentsTests { + enum KeptOffICloud: CaseIterable { + case localOnly + case sample + } + + @MainActor + private struct Dependents { + let tableFavorite: FavoriteTablesStorage.FavoriteEntry + let databaseFavorite: FavoriteDatabaseEntry + let savedQuery: SQLFavorite + let folder: SQLFavoriteFolder + let layoutKey: ColumnLayoutTableKey + + var tableFavoriteId: String { FavoriteTablesStorage.syncId(for: tableFavorite) } + var databaseFavoriteId: String { FavoriteDatabasesStorage.syncId(for: databaseFavorite) } + var layoutCategory: String { FileColumnLayoutPersister.syncCategory(for: layoutKey.storageKey) } + + var recordIDs: Set { + [ + SyncLocalOnlyDependentsTests.recordID(.tableFavorite, tableFavoriteId), + SyncLocalOnlyDependentsTests.recordID(.favoriteDatabase, databaseFavoriteId), + SyncLocalOnlyDependentsTests.recordID(.favorite, savedQuery.id.uuidString), + SyncLocalOnlyDependentsTests.recordID(.favoriteFolder, folder.id.uuidString), + SyncLocalOnlyDependentsTests.recordID(.settings, layoutCategory) + ] + } + } + + private static let zoneID = SyncTestEnvironment.zoneID + + private let environment: SyncTestEnvironment + + init() throws { + environment = try SyncTestEnvironment(label: "sync-local-only-dependents") + } + + private var tracker: SyncChangeTracker { environment.tracker } + private var metadata: SyncMetadataStorage { environment.metadata } + + private static func recordID(_ type: SyncRecordType, _ id: String) -> CKRecord.ID { + SyncRecordMapper.recordID(type: type, id: id, in: zoneID) + } + + private func addConnection(_ keptOff: KeptOffICloud? = nil) -> DatabaseConnection { + var connection = TestFixtures.makeConnection() + connection.localOnly = keptOff == .localOnly + connection.isSample = keptOff == .sample + environment.connections.addConnection(connection) + return connection + } + + private func layoutKey(_ connectionId: UUID, table: String = "orders") -> ColumnLayoutTableKey { + ColumnLayoutTableKey(connectionId: connectionId, databaseName: "shop", schemaName: "public", tableName: table) + } + + private func saveLayout(_ key: ColumnLayoutTableKey) { + var layout = ColumnLayoutState() + layout.columnWidths = ["id": 80] + environment.columnLayouts.save(layout, for: key) + } + + private func addDependents(of connectionId: UUID) async -> Dependents { + let dependents = Dependents( + tableFavorite: FavoriteTablesStorage.FavoriteEntry( + connectionId: connectionId, database: "shop", schema: "public", name: "orders" + ), + databaseFavorite: FavoriteDatabaseEntry( + connectionId: connectionId, database: "shop", environment: .production + ), + savedQuery: SQLFavorite(name: "Revenue", query: "SELECT 1", connectionId: connectionId), + folder: SQLFavoriteFolder(name: "Reports", connectionId: connectionId), + layoutKey: layoutKey(connectionId) + ) + environment.favoriteTables.addFavorite(name: "orders", schema: "public", database: "shop", connectionId: connectionId) + environment.favoriteDatabases.setFavorite(database: "shop", environment: .production, connectionId: connectionId) + #expect(await environment.favorites.addFavorite(dependents.savedQuery)) + #expect(await environment.favorites.addFolder(dependents.folder)) + saveLayout(dependents.layoutKey) + return dependents + } + + private func expectMarked(_ dependents: Dependents) { + #expect(tracker.dirtyRecords(for: .tableFavorite).contains(dependents.tableFavoriteId)) + #expect(tracker.dirtyRecords(for: .favoriteDatabase).contains(dependents.databaseFavoriteId)) + #expect(tracker.dirtyRecords(for: .favorite).contains(dependents.savedQuery.id.uuidString)) + #expect(tracker.dirtyRecords(for: .favoriteFolder).contains(dependents.folder.id.uuidString)) + #expect(tracker.dirtyRecords(for: .settings).contains(dependents.layoutCategory)) + } + + private func runCycle(_ transport: ScriptedSyncTransport) async -> SyncError? { + await environment.makeCoordinator(transport: transport).runSyncCycle() + } + + private static func isolatedQueryHistory() -> QueryHistoryManager { + QueryHistoryManager( + storage: QueryHistoryStorage( + databaseURL: FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("local-only-purge-\(UUID().uuidString).db"), + removeDatabaseOnDeinit: true + ), + isCapturePaused: { false } + ) + } + + // MARK: - Push + + @Test( + "A connection kept off iCloud keeps its favorites, saved queries, folders and layouts off it, marks held", + arguments: KeptOffICloud.allCases + ) + func keptOffConnectionHoldsItsDependents(_ keptOff: KeptOffICloud) async { + let connection = addConnection(keptOff) + let dependents = await addDependents(of: connection.id) + let transport = ScriptedSyncTransport(zoneID: Self.zoneID) + + let failure = await runCycle(transport) + + #expect(failure == nil) + #expect(await transport.pushedRecords.isEmpty) + #expect(await transport.pushedDeletions.isEmpty) + expectMarked(dependents) + } + + @Test("A synced connection's favorites, saved queries, folders and layouts go up and their marks clear") + func syncedConnectionPushesItsDependents() async { + let connection = addConnection() + let dependents = await addDependents(of: connection.id) + let transport = ScriptedSyncTransport(zoneID: Self.zoneID) + + let failure = await runCycle(transport) + + #expect(failure == nil) + #expect(Set(await transport.pushedRecords.map(\.recordID)).isSuperset(of: dependents.recordIDs)) + #expect(tracker.dirtyRecords(for: .tableFavorite).isEmpty) + #expect(tracker.dirtyRecords(for: .favoriteDatabase).isEmpty) + #expect(tracker.dirtyRecords(for: .favorite).isEmpty) + #expect(tracker.dirtyRecords(for: .favoriteFolder).isEmpty) + #expect(tracker.dirtyRecords(for: .settings).isEmpty) + } + + @Test("Deletions of a Local only connection's favorites, saved queries, folders and layouts are held") + func localOnlyDeletionsAreHeld() async { + let connection = addConnection(.localOnly) + let dependents = await addDependents(of: connection.id) + environment.favoriteTables.removeFavorite(name: "orders", schema: "public", database: "shop", connectionId: connection.id) + environment.favoriteDatabases.removeFavorite(database: "shop", connectionId: connection.id) + #expect(await environment.favorites.deleteFavorite(id: dependents.savedQuery.id)) + #expect(await environment.favorites.deleteFolder(id: dependents.folder.id)) + environment.columnLayouts.clear(for: dependents.layoutKey) + let transport = ScriptedSyncTransport(zoneID: Self.zoneID) + + let failure = await runCycle(transport) + + #expect(failure == nil) + #expect(await transport.pushedDeletions.isEmpty) + #expect(tracker.tombstonedIds(for: .tableFavorite) == [dependents.tableFavoriteId]) + #expect(tracker.tombstonedIds(for: .favoriteDatabase) == [dependents.databaseFavoriteId]) + #expect(tracker.tombstonedIds(for: .favorite) == [dependents.savedQuery.id.uuidString]) + #expect(tracker.tombstonedIds(for: .favoriteFolder) == [dependents.folder.id.uuidString]) + #expect(tracker.tombstonedIds(for: .settings) == [dependents.layoutCategory]) + } + + @Test("Putting a connection back in sync sends the edits and deletions held while it was Local only") + func reincludedConnectionReleasesHeldWork() async { + let connection = addConnection(.localOnly) + let kept = await addDependents(of: connection.id) + environment.favoriteTables.addFavorite(name: "gone", schema: "public", database: "shop", connectionId: connection.id) + environment.favoriteTables.removeFavorite(name: "gone", schema: "public", database: "shop", connectionId: connection.id) + let removedId = FavoriteTablesStorage.syncId(for: FavoriteTablesStorage.FavoriteEntry( + connectionId: connection.id, database: "shop", schema: "public", name: "gone" + )) + #expect(await runCycle(ScriptedSyncTransport(zoneID: Self.zoneID)) == nil) + #expect(environment.connections.mutateConnections(ids: [connection.id]) { $0.localOnly = false }) + let transport = ScriptedSyncTransport(zoneID: Self.zoneID) + + let failure = await runCycle(transport) + + #expect(failure == nil) + #expect(Set(await transport.pushedRecords.map(\.recordID)).isSuperset(of: kept.recordIDs)) + #expect(await transport.pushedDeletions == [Self.recordID(.tableFavorite, removedId)]) + #expect(tracker.tombstonedIds(for: .tableFavorite).isEmpty) + } + + @Test("A deletion recorded before deletions had owners still goes up") + func legacyOwnerlessTombstoneStillPushes() async { + metadata.addTombstone("legacy", type: .tableFavorite) + let transport = ScriptedSyncTransport(zoneID: Self.zoneID) + + let failure = await runCycle(transport) + + #expect(failure == nil) + #expect(await transport.pushedDeletions == [Self.recordID(.tableFavorite, "legacy")]) + } + + @Test("An unreadable connection list holds every record that belongs to a connection and sends the rest") + func unreadableConnectionStoreHoldsOwnedRecords() async throws { + let connection = addConnection() + environment.favoriteTables.addFavorite(name: "orders", schema: nil, database: "shop", connectionId: connection.id) + let global = SQLFavorite(name: "Everywhere", query: "SELECT 1") + #expect(await environment.favorites.addFavorite(global)) + metadata.addTombstones(["owned"], type: .favoriteDatabase, owner: connection.id) + try Data("not json".utf8).write(to: environment.directory.appendingPathComponent("connections.json")) + environment.connections.invalidateCache() + let transport = ScriptedSyncTransport(zoneID: Self.zoneID) + + let failure = await runCycle(transport) + + #expect(failure == nil) + #expect(await transport.pushedRecords.map(\.recordID) == [Self.recordID(.favorite, global.id.uuidString)]) + #expect(await transport.pushedDeletions.isEmpty) + #expect(tracker.dirtyRecords(for: .tableFavorite).count == 1) + #expect(tracker.tombstonedIds(for: .favoriteDatabase) == ["owned"]) + } + + @Test("A connection edit still goes up when the push could not tell which connections are kept off iCloud") + func connectionEditSurvivesUnknownOwners() async { + let connection = addConnection() + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + let boundary = SyncBoundary(includedTypes: Set(SyncRecordType.allCases), excludedConnectionIds: nil) + + let batch = await coordinator.collectPushBatch( + snapshot: tracker.editSnapshot(), boundary: boundary, zoneID: Self.zoneID + ) + + #expect(batch.records.map(\.recordID) == [Self.recordID(.connection, connection.id.uuidString)]) + #expect(tracker.dirtyRecords(for: .connection) == [connection.id.uuidString]) + } + + // MARK: - Column layout deletions + + @Test("A column layout cleared on a synced connection sends its deletion") + func clearedLayoutSendsItsDeletion() async { + let connection = addConnection() + let key = layoutKey(connection.id) + saveLayout(key) + #expect(await runCycle(ScriptedSyncTransport(zoneID: Self.zoneID)) == nil) + environment.columnLayouts.clear(for: key) + let transport = ScriptedSyncTransport(zoneID: Self.zoneID) + + let failure = await runCycle(transport) + + let category = FileColumnLayoutPersister.syncCategory(for: key.storageKey) + #expect(failure == nil) + #expect(await transport.pushedDeletions == [Self.recordID(.settings, category)]) + #expect(tracker.tombstonedIds(for: .settings).isEmpty) + } + + @Test("A layout deletion left over for a layout this Mac still holds is dropped rather than sent") + func staleLayoutTombstoneIsDropped() async { + let connection = addConnection() + let key = layoutKey(connection.id) + saveLayout(key) + #expect(await runCycle(ScriptedSyncTransport(zoneID: Self.zoneID)) == nil) + let category = FileColumnLayoutPersister.syncCategory(for: key.storageKey) + metadata.addTombstone(category, type: .settings) + let transport = ScriptedSyncTransport(zoneID: Self.zoneID) + + let failure = await runCycle(transport) + + #expect(failure == nil) + #expect(await transport.pushedDeletions.isEmpty) + #expect(tracker.tombstonedIds(for: .settings).isEmpty) + #expect(environment.columnLayouts.load(for: key) != nil) + } + + @Test("A layout this Mac is deleting is not brought back by a pull before the deletion goes up") + func pulledLayoutWaitingOnItsDeletionStaysDeleted() async throws { + let connection = addConnection() + let key = layoutKey(connection.id) + saveLayout(key) + let category = FileColumnLayoutPersister.syncCategory(for: key.storageKey) + let remote = SyncRecordMapper.toCKRecord( + category: category, + settingsData: Data(#"{"columnWidths":{"id":120}}"#.utf8), + in: Self.zoneID + ) + environment.columnLayouts.clear(for: key) + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + + let acknowledged = await coordinator.applyPullResult( + PullResult(changedRecords: [remote], deletedRecordIDs: [], newToken: nil) + ) + let transport = ScriptedSyncTransport(zoneID: Self.zoneID) + let failure = await runCycle(transport) + + #expect(acknowledged) + #expect(failure == nil) + #expect(environment.columnLayouts.load(for: key) == nil) + #expect(await transport.pushedDeletions == [Self.recordID(.settings, category)]) + } + + @Test("A layout deletion from another Mac that cannot be written here is not acknowledged") + func unwritableLayoutDeletionIsNotAcknowledged() async throws { + let connection = addConnection() + let removed = layoutKey(connection.id) + let kept = layoutKey(connection.id, table: "customers") + saveLayout(removed) + saveLayout(kept) + let directory = environment.directory.appendingPathComponent("ColumnLayout", isDirectory: true) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: directory.path) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: directory.path) } + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + + let acknowledged = await coordinator.applyPullResult(PullResult( + changedRecords: [], + deletedRecordIDs: [Self.recordID(.settings, FileColumnLayoutPersister.syncCategory(for: removed.storageKey))], + newToken: nil + )) + + #expect(!acknowledged) + #expect(environment.columnLayouts.load(for: removed) != nil) + #expect(tracker.dirtyRecords(for: .settings).contains(FileColumnLayoutPersister.syncCategory(for: removed.storageKey))) + } + + @Test("A layout clear that cannot be written leaves the layout and records no deletion") + func unwritableLayoutClearRecordsNoDeletion() throws { + let connection = addConnection() + let cleared = layoutKey(connection.id) + saveLayout(cleared) + saveLayout(layoutKey(connection.id, table: "customers")) + let directory = environment.directory.appendingPathComponent("ColumnLayout", isDirectory: true) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: directory.path) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: directory.path) } + + environment.columnLayouts.clear(for: cleared) + + #expect(environment.columnLayouts.load(for: cleared) != nil) + #expect(tracker.tombstonedIds(for: .settings).isEmpty) + } + + // MARK: - Deleting the connection + + @Test("Deleting a Local only connection removes its dependents without a trace in iCloud sync") + func localOnlyPurgeLeavesNoTombstones() async throws { + let connection = addConnection(.localOnly) + let dependents = await addDependents(of: connection.id) + environment.favoriteTables.addFavorite(name: "gone", schema: nil, database: "shop", connectionId: connection.id) + environment.favoriteTables.removeFavorite(name: "gone", schema: nil, database: "shop", connectionId: connection.id) + #expect(tracker.tombstonedIds(for: .tableFavorite).count == 1) + #expect(environment.connections.saveConnections([])) + let history = Self.isolatedQueryHistory() + + ConnectionLocalState.purge( + connectionIds: [connection.id], + origin: .localOnly, + tableScopedStores: [environment.columnLayouts], + favoriteTables: environment.favoriteTables, + favoriteDatabases: environment.favoriteDatabases, + sqlFavorites: environment.favorites, + queryHistory: history, + syncTracker: tracker + ) + await ConnectionLocalState.purgeAsyncStores( + [connection.id], origin: .localOnly, sqlFavorites: environment.favorites, queryHistory: history, + syncTracker: tracker + ) + let transport = ScriptedSyncTransport(zoneID: Self.zoneID) + let failure = await runCycle(transport) + + #expect(failure == nil) + #expect(await transport.pushedRecords.isEmpty) + #expect(await transport.pushedDeletions.isEmpty) + #expect(await environment.favorites.fetchFavorite(id: dependents.savedQuery.id) == nil) + #expect(environment.favoriteTables.favorites(for: connection.id).isEmpty) + #expect(tracker.ownersKeptOffSync.isEmpty) + for type in SyncRecordType.allCases { + #expect(tracker.tombstonedIds(for: type).isEmpty, "\(type.rawValue)") + #expect(tracker.dirtyRecords(for: type).isEmpty, "\(type.rawValue)") + } + } + + @Test("A deleted Local only connection stays out of iCloud until its saved queries are gone") + func deletedLocalOnlyOwnerStaysExcludedUntilItsSavedQueriesAreGone() async { + let connection = addConnection(.localOnly) + let savedQuery = SQLFavorite(name: "Revenue", query: "SELECT 1", connectionId: connection.id) + #expect(await environment.favorites.addFavorite(savedQuery)) + #expect(environment.connections.saveConnections([])) + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + let history = Self.isolatedQueryHistory() + + ConnectionLocalState.purge( + connectionIds: [connection.id], + origin: .localOnly, + tableScopedStores: [], + favoriteTables: environment.favoriteTables, + favoriteDatabases: environment.favoriteDatabases, + sqlFavorites: environment.favorites, + queryHistory: history, + syncTracker: tracker + ) + + #expect(tracker.ownersKeptOffSync == [connection.id]) + #expect(!coordinator.syncBoundary(settings: .default).includes(.favorite, owner: connection.id)) + + await ConnectionLocalState.purgeAsyncStores( + [connection.id], origin: .localOnly, sqlFavorites: environment.favorites, queryHistory: history, + syncTracker: tracker + ) + + #expect(await environment.favorites.fetchFavorite(id: savedQuery.id) == nil) + #expect(tracker.ownersKeptOffSync.isEmpty) + } + + @Test("Deleting a Local only connection forgets the deletions held for it, and a synced one keeps them") + func connectionDeleteDiscardsOnlyALocalOnlyOwnersTombstones() { + let localOnly = addConnection(.localOnly) + let synced = addConnection() + metadata.addTombstones(["held"], type: .tableFavorite, owner: localOnly.id) + metadata.addTombstones(["pending"], type: .tableFavorite, owner: synced.id) + metadata.addTombstone( + FileColumnLayoutPersister.syncCategory(for: layoutKey(localOnly.id).storageKey), + type: .settings + ) + + #expect(environment.connections.deleteConnection(localOnly)) + #expect(tracker.tombstonedIds(for: .tableFavorite) == ["pending"]) + #expect(tracker.tombstonedIds(for: .settings).isEmpty) + #expect(tracker.tombstonedIds(for: .connection).isEmpty) + + #expect(environment.connections.deleteConnections([synced])) + #expect(tracker.tombstonedIds(for: .tableFavorite) == ["pending"]) + #expect(tracker.tombstonedIds(for: .connection) == [synced.id.uuidString]) + } + + // MARK: - Pruning + + @Test("Pruning keeps a month-old deletion held for a Local only connection and drops a pushable one") + func pruningKeepsHeldTombstones() throws { + let localOnly = addConnection(.localOnly) + let synced = addConnection() + let fortyDaysAgo = Date(timeIntervalSinceNow: -60 * 60 * 24 * 40) + let tombstones = [ + Tombstone(id: "held", deletedAt: fortyDaysAgo, owner: localOnly.id), + Tombstone(id: "expired", deletedAt: fortyDaysAgo, owner: synced.id), + Tombstone(id: "recent", deletedAt: Date(), owner: synced.id) + ] + metadata.userDefaults.set( + try JSONEncoder().encode(tombstones), + forKey: "com.TablePro.sync.tombstones.\(SyncRecordType.tableFavorite.rawValue)" + ) + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + + coordinator.pruneTombstones(within: coordinator.syncBoundary(settings: SyncSettings.default)) + + #expect(tracker.tombstonedIds(for: .tableFavorite) == ["held", "recent"]) + } + + @Test("A deletion held when the push ran is not pruned by a connection put back in sync during the cycle") + func heldTombstoneSurvivesAScopeChangeMidCycle() async throws { + let connection = addConnection(.localOnly) + let fortyDaysAgo = Date(timeIntervalSinceNow: -60 * 60 * 24 * 40) + metadata.userDefaults.set( + try JSONEncoder().encode([Tombstone(id: "held", deletedAt: fortyDaysAgo, owner: connection.id)]), + forKey: "com.TablePro.sync.tombstones.\(SyncRecordType.tableFavorite.rawValue)" + ) + let global = SQLFavorite(name: "Everywhere", query: "SELECT 1") + #expect(await environment.favorites.addFavorite(global)) + let connections = environment.connections + let transport = ScriptedSyncTransport( + zoneID: Self.zoneID, + duringPush: { _ = connections.mutateConnections(ids: [connection.id]) { $0.localOnly = false } } + ) + + let failure = await runCycle(transport) + + #expect(failure == nil) + #expect(await transport.pushedDeletions.isEmpty) + #expect(tracker.tombstonedIds(for: .tableFavorite) == ["held"]) + } + + // MARK: - Remote deletions + + @Test("A database favorite removed on another Mac is removed here and takes its mark with it") + func remoteDatabaseFavoriteDeletionIsApplied() async { + let connection = addConnection() + environment.favoriteDatabases.setFavorite(database: "shop", environment: .production, connectionId: connection.id) + environment.favoriteDatabases.setFavorite(database: "kept", environment: .testing, connectionId: connection.id) + let removed = FavoriteDatabaseEntry(connectionId: connection.id, database: "shop", environment: .production) + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + + let acknowledged = await coordinator.applyPullResult(PullResult( + changedRecords: [], + deletedRecordIDs: [Self.recordID(.favoriteDatabase, FavoriteDatabasesStorage.syncId(for: removed))], + newToken: nil + )) + + #expect(acknowledged) + #expect(environment.favoriteDatabases.favorites(for: connection.id).map(\.database) == ["kept"]) + #expect(!tracker.dirtyRecords(for: .favoriteDatabase).contains(FavoriteDatabasesStorage.syncId(for: removed))) + #expect(tracker.tombstonedIds(for: .favoriteDatabase).isEmpty) + } + + @Test("Column layouts removed on another Mac are removed here, a digest-named one included") + func remoteColumnLayoutDeletionsAreApplied() async { + let connection = addConnection() + let short = layoutKey(connection.id) + let long = layoutKey(connection.id, table: String(repeating: "t", count: 300)) + let kept = layoutKey(connection.id, table: "customers") + [short, long, kept].forEach(saveLayout) + let longRecordID = Self.recordID(.settings, FileColumnLayoutPersister.syncCategory(for: long.storageKey)) + #expect(longRecordID.recordName.contains(SyncRecordName.digestPrefix)) + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + + let acknowledged = await coordinator.applyPullResult(PullResult( + changedRecords: [], + deletedRecordIDs: [ + Self.recordID(.settings, FileColumnLayoutPersister.syncCategory(for: short.storageKey)), + longRecordID + ], + newToken: nil + )) + + #expect(acknowledged) + #expect(environment.columnLayouts.load(for: short) == nil) + #expect(environment.columnLayouts.load(for: long) == nil) + #expect(environment.columnLayouts.load(for: kept) != nil) + #expect(tracker.dirtyRecords(for: .settings) == [FileColumnLayoutPersister.syncCategory(for: kept.storageKey)]) + #expect(tracker.tombstonedIds(for: .settings).isEmpty) + } + + @Test("Database favorite and column layout deletions from another Mac wait while their category is off") + func remoteDeletionsFollowTheirCategory() async { + let connection = addConnection() + environment.favoriteDatabases.setFavorite(database: "shop", environment: .production, connectionId: connection.id) + let key = layoutKey(connection.id) + saveLayout(key) + var settings = SyncSettings.default + settings.syncDatabaseFavorites = false + settings.syncSettings = false + AppSettingsStorage(userDefaults: environment.defaults).saveSync(settings) + let entry = FavoriteDatabaseEntry(connectionId: connection.id, database: "shop", environment: .production) + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + + let acknowledged = await coordinator.applyPullResult(PullResult( + changedRecords: [], + deletedRecordIDs: [ + Self.recordID(.favoriteDatabase, FavoriteDatabasesStorage.syncId(for: entry)), + Self.recordID(.settings, FileColumnLayoutPersister.syncCategory(for: key.storageKey)) + ], + newToken: nil + )) + + #expect(acknowledged) + #expect(environment.favoriteDatabases.favorites(for: connection.id).count == 1) + #expect(environment.columnLayouts.load(for: key) != nil) + } + + // MARK: - Owners + + @Test("Every store that deletes something belonging to a connection records the connection as its owner") + func dependentStoresRecordTheirOwner() async { + let connection = addConnection() + let dependents = await addDependents(of: connection.id) + let second = SQLFavorite(name: "Second", query: "SELECT 2", connectionId: connection.id) + let global = SQLFavorite(name: "Global", query: "SELECT 3") + #expect(await environment.favorites.addFavorite(second)) + #expect(await environment.favorites.addFavorite(global)) + environment.favoriteTables.addFavorite(name: "a|b", schema: "public", database: "shop", connectionId: connection.id) + let piped = FavoriteTablesStorage.FavoriteEntry( + connectionId: connection.id, database: "shop", schema: "public", name: "a|b" + ) + + environment.favoriteTables.removeFavorites(for: connection.id) + environment.favoriteDatabases.removeFavorites(for: connection.id) + #expect(await environment.favorites.deleteFavorite(id: dependents.savedQuery.id)) + await environment.favorites.deleteFavorites(ids: [second.id, global.id]) + #expect(await environment.favorites.deleteFolder(id: dependents.folder.id)) + environment.columnLayouts.clear(for: dependents.layoutKey) + + let owners = { (type: SyncRecordType) in + Dictionary(uniqueKeysWithValues: metadata.tombstones(for: type).map { ($0.id, $0.owner) }) + } + var tableFavoriteIds = [dependents.tableFavoriteId, FavoriteTablesStorage.syncId(for: piped)] + tableFavoriteIds += [FavoriteTablesStorage.legacyAlias(of: piped)].compactMap { $0 } + #expect(owners(.tableFavorite) == Dictionary(uniqueKeysWithValues: tableFavoriteIds.map { ($0, connection.id) })) + #expect(owners(.favoriteDatabase) == [dependents.databaseFavoriteId: connection.id]) + #expect(owners(.favorite) == [ + dependents.savedQuery.id.uuidString: connection.id, + second.id.uuidString: connection.id, + global.id.uuidString: UUID?.none + ]) + #expect(owners(.favoriteFolder) == [dependents.folder.id.uuidString: connection.id]) + #expect(owners(.settings) == [dependents.layoutCategory: connection.id]) + } +} diff --git a/TableProTests/Core/Sync/SyncPendingDeletionsTests.swift b/TableProTests/Core/Sync/SyncPendingDeletionsTests.swift index 089874663..8563c7743 100644 --- a/TableProTests/Core/Sync/SyncPendingDeletionsTests.swift +++ b/TableProTests/Core/Sync/SyncPendingDeletionsTests.swift @@ -131,6 +131,22 @@ struct SyncPendingDeletionsTests { #expect(on.sqlFolders == [Self.uuid]) } + @Test("A database favorite or column layout deleted on another device is withheld while its category is off") + func databaseFavoriteAndLayoutDeletionsFollowTheirSwitches() { + let layoutCategory = "columnLayout.\(Self.uuid.uuidString).shop.public.orders" + let deletions = Self.deletion(of: .favoriteDatabase, id: Self.tableFavoriteId) + + Self.deletion(of: .settings, id: layoutCategory) + let off = SyncPendingDeletions.parse(deletions, settings: Self.settings { + $0.syncDatabaseFavorites = false + $0.syncSettings = false + }) + let on = SyncPendingDeletions.parse(deletions, settings: Self.everyCategoryOn) + + #expect(off == SyncPendingDeletions()) + #expect(on.databaseFavorites == [Self.tableFavoriteId]) + #expect(on.settingsRecordNames == [SyncRecordType.settings.recordName(for: layoutCategory)]) + } + @Test("A category switched off withholds its own deletions and no other") func switchedOffCategoryLeavesOthersApplied() { let pending = SyncPendingDeletions.parse( diff --git a/docs/features/icloud-sync.mdx b/docs/features/icloud-sync.mdx index aa6f9051c..4c88444d7 100644 --- a/docs/features/icloud-sync.mdx +++ b/docs/features/icloud-sync.mdx @@ -33,7 +33,9 @@ A sync runs at launch, when you switch back to the app, and 2 seconds after you ## Keeping a connection off iCloud -A localhost or throwaway database is rarely worth a round trip. Mark it **Local only** in the connection form's **Advanced** pane, or right-click it and choose **Exclude from iCloud Sync**. They show a struck-through cloud icon in the connection list, and the flag survives duplicating and exporting. +A localhost or throwaway database is rarely worth a round trip. Mark it **Local only** in the connection form's **Advanced** pane, or right-click it and choose **Exclude from iCloud Sync**. It shows a struck-through cloud icon in the connection list, and the flag survives duplicating and exporting. + +What belongs to the connection stays on this Mac with it: its table and database favorites, the saved queries and folders scoped to it, and its column widths, order, and hidden columns. Edits and removals you make to them wait here while the connection is Local only, and the first sync after you turn it off sends them. Deleting a Local only connection removes those items from this Mac and from nowhere else. ## Checking that it worked