From 93fc1e9df6b559090a39c1fbf38f61642bb08500 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sat, 26 Sep 2026 13:08:50 +0700 Subject: [PATCH] fix(plugin-mongodb): send the connection's write concern with every shell write and name the documents a failed write already changed --- CHANGELOG.md | 5 + .../MongoDBConnection+ScriptHelpers.swift | 95 ++++- .../MongoDBPluginDriver.swift | 58 ++- .../MongoDBTimeoutPolicy.swift | 27 +- .../MongoScriptCommandBuilder.swift | 165 ++++++-- .../MongoScriptContext.swift | 18 + .../MongoScriptError.swift | 18 +- .../MongoScriptExport.swift | 6 +- .../MongoDBDriverPlugin/MongoScriptHost.swift | 148 ++++++-- .../MongoDBDriverPlugin/MongoScriptJson.swift | 8 + .../MongoScriptPrelude.swift | 42 ++- .../MongoScriptRuntime.swift | 21 +- .../MongoDBDriverPlugin/MongoScriptText.swift | 37 ++ .../MongoWriteFailure.swift | 61 ++- .../MongoWriteLedger.swift | 131 +++++++ TablePro/Resources/Localizable.xcstrings | 15 + .../MongoDB/MongoDBTimeoutPolicyTests.swift | 45 +++ .../MongoDB/MongoScriptCommandTests.swift | 236 +++++++++++- .../MongoDB/MongoScriptPreludeTests.swift | 173 +++++++++ .../Plugins/MongoWriteFailureTests.swift | 110 +++++- .../Plugins/MongoWriteLedgerTests.swift | 354 ++++++++++++++++++ docs/databases/mongodb.mdx | 26 +- project.yml | 1 + scripts/check-mongodb-interruption-codes.sh | 123 ++++++ 24 files changed, 1789 insertions(+), 134 deletions(-) create mode 100644 Plugins/MongoDBDriverPlugin/MongoWriteLedger.swift create mode 100644 TableProTests/Plugins/MongoWriteLedgerTests.swift create mode 100755 scripts/check-mongodb-interruption-codes.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index fd43cf0c69..884d8866c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,6 +131,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pre-connect script failures sometimes reported without the script's own error message. - Failed MongoDB statements, including writes the server rejected, reported as successful with an empty result. +- MongoDB **Write Concern** setting ignored by every write except inserts, data grid saves included. +- `writeConcern` option ignored by MongoDB shell writes, and `ordered` by `insertMany`. +- `remove(filter, {justOne: true})` in the MongoDB shell deleting every matching document. +- MongoDB write that stopped part-way reported without saying documents had already changed. +- Empty error message when a MongoDB `insertMany` stopped at an oversized document. - `tablepro-mcp` crashing when its standard input was non-blocking. - `tablepro-mcp` using a full CPU core, or crashing, when its standard output or error was non-blocking. - Server connections piling up while browsing many databases or schemas, and staying open after a failed connect. (#3103) diff --git a/Plugins/MongoDBDriverPlugin/MongoDBConnection+ScriptHelpers.swift b/Plugins/MongoDBDriverPlugin/MongoDBConnection+ScriptHelpers.swift index a8ab73b585..4e6a676bc7 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBConnection+ScriptHelpers.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBConnection+ScriptHelpers.swift @@ -32,6 +32,45 @@ import os extension MongoDBConnection { func scriptRunCommand(client: OpaquePointer, command: String, database: String?) throws -> String { + let result = try sendCommand(client: client, command: command, database: database) + try checkCancelled() + guard result.ok else { throw makeError(result.error) } + return result.reply + } + + /// Runs a write command and keeps its reply whatever the answer. + /// + /// Unlike `scriptRunCommand`, this does not check for a cancel once the server has answered: the + /// write has happened by then, and throwing its reply away would lose the only record of what it + /// changed. The host's cancel latch refuses the script's next call instead. + func scriptWriteCommand(client: OpaquePointer, command: String, database: String?) throws -> MongoWriteOutcome { + let result = try sendCommand(client: client, command: command, database: database) + let failure = MongoWriteFailure.read(fromReply: result.reply) + guard !result.ok, failure == nil else { + return MongoWriteOutcome(replyJson: result.reply, failure: failure) + } + return MongoWriteOutcome(replyJson: result.reply, failure: unansweredFailure(result.error)) + } + + /// The write concern the connection's URI sets, as the `writeConcern` document a command takes, + /// or nil when it sets none and the server's default applies. + /// + /// Read from libmongoc rather than from the connection's setting, so `journal` and + /// `wtimeoutMS` from an imported connection string travel with `w`. + func writeConcernJson(client: OpaquePointer) -> String? { + guard let concern = mongoc_client_get_write_concern(client), + !mongoc_write_concern_is_default(concern), + let copy = mongoc_write_concern_copy(concern) else { return nil } + defer { mongoc_write_concern_destroy(copy) } + let document = bson_new() + defer { bson_destroy(document) } + guard mongoc_write_concern_append(copy, document), let json = bsonToJson(document) else { return nil } + return MongoScriptJson.member(of: json, key: "writeConcern") + } + + private func sendCommand( + client: OpaquePointer, command: String, database: String? + ) throws -> (ok: Bool, reply: String, error: bson_error_t) { try checkCancelled() guard let bsonCommand = jsonToBson(command) else { @@ -52,10 +91,18 @@ extension MongoDBConnection { let ok = resolved.withCString { mongoc_client_command_simple(client, $0, bsonCommand, nil, reply, &error) } + return (ok, bsonToJson(reply) ?? "{}", error) + } - try checkCancelled() - guard ok else { throw makeError(error) } - return bsonToJson(reply) ?? "{}" + /// A failed call whose reply carries no server answer. A stream or protocol error means the + /// command may have reached the server before the connection broke; anything else stopped it + /// on this side. + private func unansweredFailure(_ error: bson_error_t) -> MongoWriteFailure { + let reported = makeError(error) + let reachedServer = error.domain == MONGOC_ERROR_STREAM.rawValue || error.domain == MONGOC_ERROR_PROTOCOL.rawValue + return MongoWriteFailure( + code: reported.code, message: reported.message, stage: reachedServer ? .unanswered : .notSent + ) } func scriptFind( @@ -139,14 +186,20 @@ extension MongoDBConnection { /// A document with no `_id` gets one prepended through libbson rather than through a Swift /// dictionary, so the rest of its fields keep the order the script wrote them in and `_id` /// lands first, where the server puts it. + /// + /// A failed insert still returns: an ordered insert keeps every document before the one that + /// failed, and the reply's `insertedCount` is the only record of how many. func scriptInsert( client: OpaquePointer, database: String, collection: String, - documents: [String] - ) throws -> [String] { + documents: [String], + options: String? + ) throws -> (identifiers: [String], outcome: MongoWriteOutcome) { try checkCancelled() - guard !documents.isEmpty else { return [] } + guard !documents.isEmpty else { + return ([], MongoWriteOutcome(replyJson: "{}", failure: nil)) + } let handle = try getCollection(client, database: database, collection: collection) defer { mongoc_collection_destroy(handle) } @@ -174,6 +227,14 @@ extension MongoDBConnection { identifiers.append("{\"$oid\": \"\(hex)\"}") } + let optsBson = try options.map { json -> OpaquePointer in + guard let parsed = jsonToBson(json) else { + throw MongoDBError(code: 0, message: MongoScriptText.invalidDocument(json)) + } + return parsed + } + defer { if let optsBson { bson_destroy(optsBson) } } + try checkCancelled() var pointers: [OpaquePointer?] = prepared.map { Optional($0) } @@ -183,10 +244,26 @@ extension MongoDBConnection { let ok = pointers.withUnsafeMutableBufferPointer { buffer -> Bool in guard let base = buffer.baseAddress else { return false } - return mongoc_collection_insert_many(handle, base, buffer.count, nil, reply, &error) + return mongoc_collection_insert_many(handle, base, buffer.count, optsBson, reply, &error) + } + let replyJson = bsonToJson(reply) ?? "{}" + guard !ok else { + return (identifiers, MongoWriteOutcome(replyJson: replyJson, failure: nil)) } - guard ok else { throw makeError(error) } - return identifiers + let failure = MongoWriteFailure.read(fromReply: replyJson) ?? insertFailure(error) + return (identifiers, MongoWriteOutcome(replyJson: replyJson, failure: failure)) + } + + /// An insert that failed with no server answer. + /// + /// libmongoc 1.28.1 sends a large insert in batches, and clears its error when the last batch + /// it sent succeeded. An insert that stops at a document over the server's size limit after a + /// batch already went therefore fails with no error at all, while its reply counts that batch. + private func insertFailure(_ error: bson_error_t) -> MongoWriteFailure { + guard error.domain == 0, error.code == 0 else { return unansweredFailure(error) } + return MongoWriteFailure( + code: 0, message: MongoScriptText.insertStoppedAtOversizedDocument, stage: .stoppedBetweenBatches + ) } func listIndexesJsonSync( diff --git a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 5c6da17ebe..dace0b6335 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -223,11 +223,8 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ), rowCap: rowCap ) - } catch let failure as MongoScriptStatementFailure { - currentDb = failure.databaseSwitch - throw mapExecutionError(failure.underlying) } catch { - throw mapExecutionError(error) + throw reportedError(error) } } @@ -275,16 +272,32 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) } - private func mapExecutionError(_ error: Error) -> Error { - guard let mongoError = error as? MongoDBError, - MongoDBTimeoutPolicy.isTimeoutCode(mongoError.code), - let maxTimeMS = mongoConnection?.effectiveMaxTimeMS(background: false) else { - return error + /// The error a failed statement surfaces, built in one place so the timeout wording and the + /// note about documents already written cannot overwrite each other. + /// + /// A cancel stays a cancel even when the statement had written: the app discards the result of + /// a query the user stopped, so there is nothing to show the note on. A write's failure leaves + /// as a `MongoDBError` like every other, so the app reads its code the same way. + private func reportedError(_ error: Error) -> Error { + var underlying = error + var writes = MongoWriteLedger() + if let failure = error as? MongoScriptStatementFailure { + if let switched = failure.databaseSwitch { currentDb = switched } + underlying = failure.underlying + writes = failure.writes } - return MongoDBError( - code: mongoError.code, - message: MongoDBTimeoutPolicy.timeoutMessage(maxTimeMS: maxTimeMS) + if underlying is CancellationError { return underlying } + let failedWrite = underlying as? MongoWriteFailure + let code = failedWrite?.code ?? (underlying as? MongoDBError)?.code ?? 0 + let message = failedWrite?.message ?? (underlying as? MongoDBError)?.message ?? underlying.localizedDescription + let reported = writes.reportedMessage( + code: code, + message: message, + failedWrite: failedWrite?.stage, + maxTimeMS: mongoConnection?.effectiveMaxTimeMS(background: false) ) + guard reported != message || failedWrite != nil else { return underlying } + return MongoDBError(code: code, message: reported) } // MARK: - Query Cancellation @@ -838,7 +851,8 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let work = Task { do { switch try await runtime.exportPlan(for: trimmed, database: db) { - case .cursor(let plan): + case .cursor(let plan, let databaseSwitch, let writes): + if let databaseSwitch { self.currentDb = databaseSwitch } let inner = plan.isFind ? conn.streamFind( database: plan.database, collection: plan.collection, @@ -852,19 +866,23 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { pipeline: plan.pipeline, optionsJson: plan.options.aggregateOptionsJson(timeoutMS: timeout) ) - for try await element in inner { - try Task.checkCancellation() - continuation.yield(element) + do { + for try await element in inner { + try Task.checkCancellation() + continuation.yield(element) + } + } catch { + throw MongoScriptStatementFailure.carrying( + error, databaseSwitch: databaseSwitch, writes: writes + ) } case .result(let outcome): + if let switched = outcome.databaseSwitch { self.currentDb = switched } self.yieldMaterialised(outcome, into: continuation) } continuation.finish() - } catch let failure as MongoScriptStatementFailure { - self.currentDb = failure.databaseSwitch - continuation.finish(throwing: failure.underlying) } catch { - continuation.finish(throwing: error) + continuation.finish(throwing: self.reportedError(error)) } } // A consumer that stops reading has to stop the cursor too, or it keeps draining the diff --git a/Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift b/Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift index 74b0e3723b..1410241976 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift @@ -10,6 +10,15 @@ enum MongoDBServerErrorCode { static let indexNotFound: UInt32 = 27 static let maxTimeMSExpired: UInt32 = 50 static let cursorKilled: UInt32 = 237 + + /// Every code in the server's `Interruption` category on each release branch from 4.0 through + /// 9.0, read from its `src/mongo/base/error_codes.yml` (`error_codes.err` before 4.4). The + /// server fails a whole write batch with `ok: 0` for these, keeping the documents it already + /// wrote. `scripts/check-mongodb-interruption-codes.sh` diffs the set against every branch. + static let interruptionCategory: Set = [ + 24, maxTimeMSExpired, cursorKilled, 262, 279, 281, 282, 290, 355, 453, 471, 473, 485, 509, + 11_600, 11_601, 11_602, 46_841, 91_331, 10_045_600 + ] } enum MongoDBTimeoutPolicy { @@ -34,8 +43,7 @@ enum MongoDBTimeoutPolicy { } static func timeoutMessage(maxTimeMS: Int32) -> String { - let seconds = max(1, Int((Double(maxTimeMS) / 1_000).rounded())) - return String( + String( format: String( localized: """ The query timed out after %d seconds. Sorting or filtering on a field with no index \ @@ -43,7 +51,20 @@ enum MongoDBTimeoutPolicy { Add an index for that field, or raise the query timeout in Settings. """ ), - seconds + seconds(maxTimeMS) + ) + } + + static func writeTimeoutMessage(maxTimeMS: Int32) -> String { + String( + format: String( + localized: "The write did not finish within %d seconds, so MongoDB stopped it. Raise the query timeout in Settings if it needs longer." + ), + seconds(maxTimeMS) ) } + + private static func seconds(_ maxTimeMS: Int32) -> Int { + max(1, Int((Double(maxTimeMS) / 1_000).rounded())) + } } diff --git a/Plugins/MongoDBDriverPlugin/MongoScriptCommandBuilder.swift b/Plugins/MongoDBDriverPlugin/MongoScriptCommandBuilder.swift index ab29d13618..796b7014ae 100644 --- a/Plugins/MongoDBDriverPlugin/MongoScriptCommandBuilder.swift +++ b/Plugins/MongoDBDriverPlugin/MongoScriptCommandBuilder.swift @@ -2,27 +2,69 @@ import Foundation /// Builds the database commands a script's collection methods stand for. /// -/// Writes go out as commands rather than through the collection convenience calls so the whole -/// server reply is available: `n`, `nModified` and `upserted` are what a mongosh result object is -/// made of, and the convenience calls report only one of them. +/// Updates and deletes go out as commands rather than through libmongoc's collection calls, for +/// two reasons measured against libmongoc 1.28.1. The collection calls make the caller choose +/// between an update and a replacement and check the keys first, where a legacy +/// `update(filter, document)` leaves that to the server. And the Bulk API refuses `maxTimeMS` as an +/// option, which is how the query timeout reaches a write. +/// +/// `mongoc_client_command_simple` applies no write concern to a command, so every write command +/// carries the one `writeConcern(statementOptions:connectionDefault:)` resolves, the way mongosh +/// does: the statement's own if it names one, otherwise the connection's. +/// +/// A statement's write concern is rebuilt from the names the server reads rather than passed on as +/// written. mongosh takes `journal` and `wtimeoutMS` beside `j` and `wtimeout`, and `fsync` for +/// `j`. MongoDB 7.0.43 refuses a command whose write concern carries any other name +/// (`IDLUnknownField`), and libmongoc 1.28.1 drops those names from an insert's options along with +/// the connection's own write concern, so the insert goes out with neither. enum MongoScriptCommandBuilder { - enum BulkKind { - case insert - case update - case delete - } - struct BulkStatement { - let kind: BulkKind + let kind: MongoWriteOperation + let touchesMany: Bool let document: String } + /// The statement's write concern if it names one, otherwise the connection's. + /// + /// A write concern that names none of `w`, `j` and `wtimeout`, in any spelling, counts as unset, + /// as mongosh treats it: `{}` falls back to the connection's rather than dropping it. One that + /// names any of them replaces the connection's whole, again as in mongosh. + static func writeConcern(statementOptions: String?, connectionDefault: String?) -> String? { + statementWriteConcern(statementOptions) ?? connectionDefault + } + + /// The options the insert call takes from the statement. The connection's write concern is not + /// among them: the collection inherits it from the client, and libmongoc lets the statement's + /// own win. + static func insertOptions(statementOptions: String?) -> String? { + guard let statementOptions else { return nil } + let fields = [ + insertWriteConcern(statementOptions).map { "\"writeConcern\": \($0)" }, + presentMember(of: statementOptions, key: "ordered").map { "\"ordered\": \($0)" } + ].compactMap { $0 } + return fields.isEmpty ? nil : "{\(fields.joined(separator: ", "))}" + } + + /// Whether the server answers a write sent with this write concern. + /// + /// Measured on MongoDB 7.0.43: with `w: 0` and no `j: true`, an insert, update or delete command + /// is answered with `n: 0` whatever it changed, and a duplicate key or an immutable `_id` it + /// hit is not reported at all. With `j: true` beside `w: 0` it is answered in full, which is also + /// what libmongoc's own `mongoc_write_concern_is_acknowledged` says. libmongoc sends an insert + /// with `w: -1`, its legacy value for ignoring errors, without waiting for an answer, and the + /// server drops that insert, since it refuses a `w` below 0. + static func isAcknowledged(writeConcern: String?) -> Bool { + guard let writeConcern, let w = numericW(of: writeConcern), w <= 0 else { return true } + return presentMember(of: writeConcern, key: "j") == "true" + } + static func update( collection: String, filter: String, update: String, multi: Bool, - options: [String: Any] + options: [String: Any], + writeConcern: String? ) -> String { var fields = [ "\"q\": \(filter)", @@ -31,17 +73,25 @@ enum MongoScriptCommandBuilder { "\"upsert\": \(options["upsert"] as? Bool ?? false)" ] appendPassThrough(&fields, options: options, keys: ["arrayFilters", "hint", "collation"]) - return """ - {"update": \(MongoScriptJson.jsonString(collection)), "updates": [{\(fields.joined(separator: ", "))}]} - """ + return command( + ["\"update\": \(MongoScriptJson.jsonString(collection))", "\"updates\": [{\(fields.joined(separator: ", "))}]"], + writeConcern: writeConcern + ) } - static func delete(collection: String, filter: String, multi: Bool, options: [String: Any]) -> String { + static func delete( + collection: String, + filter: String, + multi: Bool, + options: [String: Any], + writeConcern: String? + ) -> String { var fields = ["\"q\": \(filter)", "\"limit\": \(multi ? 0 : 1)"] appendPassThrough(&fields, options: options, keys: ["hint", "collation"]) - return """ - {"delete": \(MongoScriptJson.jsonString(collection)), "deletes": [{\(fields.joined(separator: ", "))}]} - """ + return command( + ["\"delete\": \(MongoScriptJson.jsonString(collection))", "\"deletes\": [{\(fields.joined(separator: ", "))}]"], + writeConcern: writeConcern + ) } static func findAndModify( @@ -49,7 +99,8 @@ enum MongoScriptCommandBuilder { filter: String, update: String?, remove: Bool, - options: [String: Any] + options: [String: Any], + writeConcern: String? ) -> String { var fields = [ "\"findAndModify\": \(MongoScriptJson.jsonString(collection))", @@ -66,7 +117,7 @@ enum MongoScriptCommandBuilder { if let projection = jsonText(options["projection"]) { fields.append("\"fields\": \(projection)") } - return "{\(fields.joined(separator: ", "))}" + return command(fields, writeConcern: writeConcern) } static func createIndex(collection: String, keys: String, options: [String: Any]) -> String { @@ -120,14 +171,16 @@ enum MongoScriptCommandBuilder { return "{\(fields.joined(separator: ", "))}" } - static func bulkOperation(_ operation: String, collection: String) throws -> BulkStatement { + static func bulkOperation(_ operation: String, collection: String, writeConcern: String?) throws -> BulkStatement { if let document = MongoScriptJson.member(of: operation, key: "insertOne") { let payload = MongoScriptJson.member(of: document, key: "document") ?? "{}" return BulkStatement( kind: .insert, - document: """ - {"insert": \(MongoScriptJson.jsonString(collection)), "documents": [\(payload)]} - """ + touchesMany: false, + document: command( + ["\"insert\": \(MongoScriptJson.jsonString(collection))", "\"documents\": [\(payload)]"], + writeConcern: writeConcern + ) ) } for name in ["updateOne", "updateMany", "replaceOne"] { @@ -136,12 +189,14 @@ enum MongoScriptCommandBuilder { let change = MongoScriptJson.member(of: body, key: name == "replaceOne" ? "replacement" : "update") return BulkStatement( kind: .update, + touchesMany: name == "updateMany", document: update( collection: collection, filter: filter, update: change ?? "{}", multi: name == "updateMany", - options: MongoScriptJson.options(body) + options: MongoScriptJson.options(body), + writeConcern: writeConcern ) ) } @@ -149,11 +204,13 @@ enum MongoScriptCommandBuilder { guard let body = MongoScriptJson.member(of: operation, key: name) else { continue } return BulkStatement( kind: .delete, + touchesMany: name == "deleteMany", document: delete( collection: collection, filter: MongoScriptJson.member(of: body, key: "filter") ?? "{}", multi: name == "deleteMany", - options: [:] + options: MongoScriptJson.options(body), + writeConcern: writeConcern ) ) } @@ -191,6 +248,62 @@ enum MongoScriptCommandBuilder { return false } + private static func command(_ fields: [String], writeConcern: String?) -> String { + guard let writeConcern else { return "{\(fields.joined(separator: ", "))}" } + return "{\((fields + ["\"writeConcern\": \(writeConcern)"]).joined(separator: ", "))}" + } + + private typealias ConcernField = (name: String, value: String) + + private static func statementWriteConcern(_ statementOptions: String?) -> String? { + statementConcernFields(statementOptions).map(concernDocument) + } + + /// The statement's write concern as the insert call can take it. + /// + /// libmongoc 1.28.1 refuses `j: true` beside `w: 0` for an insert (`Invalid writeConcern`). The + /// server answers that pair in full and waits for the journal, as it does for `w: 1, j: true`: + /// it leaves out the reply only for a `w` below 1 with neither `j` nor `fsync`. So the insert + /// goes out with `w: 1`, which is what `isAcknowledged` already says the pair means. + private static func insertWriteConcern(_ statementOptions: String) -> String? { + guard let fields = statementConcernFields(statementOptions) else { return nil } + let concern = concernDocument(fields) + guard numericW(of: concern) == 0, presentMember(of: concern, key: "j") == "true" else { return concern } + return concernDocument(fields.map { $0.name == "w" ? (name: "w", value: "1") : $0 }) + } + + /// The statement's write concern under the names the server reads, or nil when it names none. + /// + /// Each name takes the first spelling the statement set, in the order mongosh reads them: + /// `j`, then `journal`, then `fsync`, and `wtimeout`, then `wtimeoutMS`. + private static func statementConcernFields(_ statementOptions: String?) -> [ConcernField]? { + guard let concern = statementOptions.flatMap({ presentMember(of: $0, key: "writeConcern") }) else { + return nil + } + let spellings = [("w", ["w"]), ("j", ["j", "journal", "fsync"]), ("wtimeout", ["wtimeout", "wtimeoutMS"])] + let fields = spellings.compactMap { name, keys -> ConcernField? in + keys.lazy.compactMap { presentMember(of: concern, key: $0) }.first.map { (name: name, value: $0) } + } + return fields.isEmpty ? nil : fields + } + + private static func concernDocument(_ fields: [ConcernField]) -> String { + "{\(fields.map { "\"\($0.name)\": \($0.value)" }.joined(separator: ", "))}" + } + + /// The write concern's `w` when it is a number rather than `"majority"` or a tag. + private static func numericW(of concern: String) -> Int64? { + guard let w = presentMember(of: concern, key: "w"), w != "true", w != "false" else { return nil } + return MongoScriptJson.number(in: concern, key: "w") + } + + /// A member the statement set, where `null` counts as not set, as it does in mongosh: sending it + /// on makes the server refuse the whole command. + private static func presentMember(of json: String, key: String) -> String? { + guard let value = MongoScriptJson.member(of: json, key: key), value != "null" else { return nil } + return value + } + private static func appendPassThrough(_ fields: inout [String], options: [String: Any], keys: [String]) { for key in keys { guard let text = jsonText(options[key]) else { continue } diff --git a/Plugins/MongoDBDriverPlugin/MongoScriptContext.swift b/Plugins/MongoDBDriverPlugin/MongoScriptContext.swift index 8e531535d5..86d415454b 100644 --- a/Plugins/MongoDBDriverPlugin/MongoScriptContext.swift +++ b/Plugins/MongoDBDriverPlugin/MongoScriptContext.swift @@ -33,4 +33,22 @@ enum MongoScriptContext { } return context } + + /// The failed write an exception stands for, or nil when it is not a write's failure. + /// + /// Read from the exception that actually escaped rather than from the last write that failed: + /// a script can catch a write's timeout and then time out on a read, and both carry the same + /// code and message. + static func writeFailure(in exception: JSValue) -> MongoWriteFailure? { + guard exception.objectForKeyedSubscript("isMongoError")?.toBool() == true, + let stageName = exception.objectForKeyedSubscript("__writeStage"), stageName.isString, + let stage = MongoWriteFailure.Stage(rawValue: stageName.toString()) else { + return nil + } + return MongoWriteFailure( + code: UInt32(max(0, exception.objectForKeyedSubscript("code")?.toInt32() ?? 0)), + message: exception.objectForKeyedSubscript("message")?.toString() ?? "", + stage: stage + ) + } } diff --git a/Plugins/MongoDBDriverPlugin/MongoScriptError.swift b/Plugins/MongoDBDriverPlugin/MongoScriptError.swift index 8c6b7ca945..1df4bbf206 100644 --- a/Plugins/MongoDBDriverPlugin/MongoScriptError.swift +++ b/Plugins/MongoDBDriverPlugin/MongoScriptError.swift @@ -15,20 +15,26 @@ struct MongoScriptError: Error, LocalizedError, Equatable { } } -/// A statement that failed after it had already switched database. +/// A statement that failed after it had already changed something outside itself. /// /// `use("b")` rebinds the shell's `db` and the host the moment it runs, so a statement that goes on /// to throw has still moved the shell. The driver has to move with it, or its idea of the current /// database stays behind and the next save for the old database skips the rebind and runs against /// the new one. +/// +/// Writes are the other thing a failure cannot take back, so the documents the statement had +/// already changed travel with the error to the one place its message is built. struct MongoScriptStatementFailure: Error, LocalizedError { let underlying: Error - let databaseSwitch: String + let databaseSwitch: String? + let writes: MongoWriteLedger - var errorDescription: String? { underlying.localizedDescription } + var errorDescription: String? { + writes.reportedMessage(code: 0, message: underlying.localizedDescription, failedWrite: nil, maxTimeMS: nil) + } - static func carrying(_ error: Error, databaseSwitch: String?) -> Error { - guard let databaseSwitch else { return error } - return MongoScriptStatementFailure(underlying: error, databaseSwitch: databaseSwitch) + static func carrying(_ error: Error, databaseSwitch: String?, writes: MongoWriteLedger) -> Error { + guard databaseSwitch != nil || !writes.isEmpty else { return error } + return MongoScriptStatementFailure(underlying: error, databaseSwitch: databaseSwitch, writes: writes) } } diff --git a/Plugins/MongoDBDriverPlugin/MongoScriptExport.swift b/Plugins/MongoDBDriverPlugin/MongoScriptExport.swift index f8cb9bf762..4942f7775b 100644 --- a/Plugins/MongoDBDriverPlugin/MongoScriptExport.swift +++ b/Plugins/MongoDBDriverPlugin/MongoScriptExport.swift @@ -5,7 +5,11 @@ import Foundation /// The distinction exists so a statement is never evaluated twice. A cursor has not touched the /// server yet, so the export can page through it; anything else has already run, and running it /// again to fill the stream would repeat its write. +/// +/// The statement behind a cursor can still have run writes, or a `use`, before it built one, and the +/// stream can fail after the evaluation has returned. So a cursor carries what the statement did, +/// and a failed stream reports it the way a failed statement does. enum MongoScriptExport: Sendable { - case cursor(MongoScriptCursorPlan) + case cursor(MongoScriptCursorPlan, databaseSwitch: String?, writes: MongoWriteLedger) case result(MongoScriptStatementResult) } diff --git a/Plugins/MongoDBDriverPlugin/MongoScriptHost.swift b/Plugins/MongoDBDriverPlugin/MongoScriptHost.swift index 637f5bc8c1..dd345a1fac 100644 --- a/Plugins/MongoDBDriverPlugin/MongoScriptHost.swift +++ b/Plugins/MongoDBDriverPlugin/MongoScriptHost.swift @@ -19,6 +19,7 @@ final class MongoScriptHost { private let activityLock = NSLock() private var activity = Date() private var cancelled = false + private var ledger = MongoWriteLedger() private(set) var database: String private(set) var printedLines: [String] = [] private(set) var databaseSwitch: String? @@ -70,6 +71,20 @@ final class MongoScriptHost { return cancelled } + /// What the current statement has written so far. Read from the watchdog's thread as well as + /// the engine's, hence the lock. + var writes: MongoWriteLedger { + activityLock.lock() + defer { activityLock.unlock() } + return ledger + } + + private func updateLedger(_ change: (inout MongoWriteLedger) -> Void) { + activityLock.lock() + change(&ledger) + activityLock.unlock() + } + /// Clears what belonged to the previous statement. /// /// Cursors are pruned rather than dropped: a shell lets you keep one in a variable and read it @@ -80,6 +95,7 @@ final class MongoScriptHost { databaseSwitch = nil activityLock.lock() cancelled = false + ledger = MongoWriteLedger() activityLock.unlock() touch() } @@ -92,6 +108,7 @@ final class MongoScriptHost { cursors.removeAll() printedLines.removeAll() databaseSwitch = nil + updateLedger { $0 = MongoWriteLedger() } touch() self.database = database self.valueCeiling = valueCeiling @@ -128,6 +145,8 @@ final class MongoScriptHost { } catch is CancellationError { markCancelled() return MongoScriptJson.failure(message: MongoScriptText.cancelled, code: 0) + } catch let failedWrite as MongoWriteFailure { + return MongoScriptJson.failure(failedWrite) } catch let error as MongoDBError { return MongoScriptJson.failure(message: error.message, code: error.code) } catch let error as MongoScriptError { @@ -364,12 +383,57 @@ final class MongoScriptHost { } } - private func writeCommand(_ document: String, _ request: [String: Any]) throws -> String { - let reply = try command(document, request) - if let failure = MongoWriteFailure.read(fromReply: reply) { - throw MongoDBError(code: failure.code, message: failure.message) + /// The write concern a write goes out with: the statement's own, or else the connection's. + private func writeConcern(for request: [String: Any]) throws -> String? { + let connectionDefault = try withClient { connection.writeConcernJson(client: $0) } + return MongoScriptCommandBuilder.writeConcern( + statementOptions: MongoScriptJson.rawJson(request["options"]), + connectionDefault: connectionDefault + ) + } + + /// Sends one write command, and hands back the server's reply, or only the fact that the write + /// was not acknowledged when its reply cannot say what it did. + private func write( + _ operation: MongoWriteOperation, + touchesMany: Bool, + acknowledged: Bool, + command document: String, + request: [String: Any] + ) throws -> String { + let outcome = try recordingWrite(operation, touchesMany: touchesMany, acknowledged: acknowledged) { + try withClient { + try connection.scriptWriteCommand(client: $0, command: document, database: databaseName(request)) + } + } + return acknowledged ? outcome.replyJson : Self.unacknowledgedReply + } + + private static let unacknowledgedReply = "{\"acknowledged\": false}" + + /// Runs one write and enters it in the statement's ledger, then fails the call if the write + /// failed. The failure reaches the script with the server's own message and code, and with the + /// stage that marks it as a write's. + @discardableResult + private func recordingWrite( + _ operation: MongoWriteOperation, + touchesMany: Bool, + acknowledged: Bool, + send: () throws -> MongoWriteOutcome + ) throws -> MongoWriteOutcome { + updateLedger { $0.beginWrite() } + let outcome: MongoWriteOutcome + do { + outcome = try send() + } catch { + updateLedger { $0.abandonWrite() } + throw error } - return reply + updateLedger { + $0.record(operation, touchesMany: touchesMany, acknowledged: acknowledged, outcome: outcome) + } + if let failure = outcome.failure { throw failure } + return outcome } private func countDocuments(_ request: [String: Any]) throws -> String { @@ -412,49 +476,76 @@ final class MongoScriptHost { } else { documents = [MongoScriptJson.rawJson(request["document"]) ?? "{}"] } - let inserted = try withClient { - try connection.scriptInsert( - client: $0, - database: databaseName(request), - collection: collectionName(request), - documents: documents - ) + let statementOptions = MongoScriptJson.rawJson(request["options"]) + let options = MongoScriptCommandBuilder.insertOptions(statementOptions: statementOptions) + let acknowledged = try MongoScriptCommandBuilder.isAcknowledged(writeConcern: writeConcern(for: request)) + var inserted: [String] = [] + try recordingWrite(.insert, touchesMany: documents.count > 1, acknowledged: acknowledged) { + let result = try withClient { + try connection.scriptInsert( + client: $0, + database: databaseName(request), + collection: collectionName(request), + documents: documents, + options: options + ) + } + inserted = result.identifiers + return result.outcome + } + let ids = inserted.joined(separator: ",") + guard acknowledged else { + return "{\"acknowledged\": false, \"insertedIds\": [\(ids)]}" } - let ids = inserted.map { $0 }.joined(separator: ",") return "{\"insertedIds\": [\(ids)], \"insertedCount\": \(inserted.count)}" } private func update(_ request: [String: Any], isReplace: Bool) throws -> String { - let options = MongoScriptJson.options(request["options"]) + let multi = !isReplace && (request["multi"] as? Bool ?? false) + let concern = try writeConcern(for: request) let statement = MongoScriptCommandBuilder.update( collection: collectionName(request), filter: MongoScriptJson.rawJson(request["filter"]) ?? "{}", update: MongoScriptJson.rawJson(request["update"]) ?? "{}", - multi: !isReplace && (request["multi"] as? Bool ?? false), - options: options + multi: multi, + options: MongoScriptJson.options(request["options"]), + writeConcern: concern + ) + return try write( + .update, touchesMany: multi, acknowledged: MongoScriptCommandBuilder.isAcknowledged(writeConcern: concern), + command: statement, request: request ) - return try writeCommand(statement, request) } private func delete(_ request: [String: Any]) throws -> String { + let multi = request["multi"] as? Bool ?? false + let concern = try writeConcern(for: request) let statement = MongoScriptCommandBuilder.delete( collection: collectionName(request), filter: MongoScriptJson.rawJson(request["filter"]) ?? "{}", - multi: request["multi"] as? Bool ?? false, - options: MongoScriptJson.options(request["options"]) + multi: multi, + options: MongoScriptJson.options(request["options"]), + writeConcern: concern + ) + return try write( + .delete, touchesMany: multi, acknowledged: MongoScriptCommandBuilder.isAcknowledged(writeConcern: concern), + command: statement, request: request ) - return try writeCommand(statement, request) } + /// Unlike an insert, update or delete, a `findAndModify` sent with `w: 0` is answered in full, + /// its document and its count included, so its reply is read whatever the write concern. private func findAndModify(_ request: [String: Any]) throws -> String { + let concern = try writeConcern(for: request) let statement = MongoScriptCommandBuilder.findAndModify( collection: collectionName(request), filter: MongoScriptJson.rawJson(request["filter"]) ?? "{}", update: MongoScriptJson.rawJson(request["update"]), remove: request["remove"] as? Bool ?? false, - options: MongoScriptJson.options(request["options"]) + options: MongoScriptJson.options(request["options"]), + writeConcern: concern ) - return try writeCommand(statement, request) + return try write(.findAndModify, touchesMany: false, acknowledged: true, command: statement, request: request) } private func bulkWrite(_ request: [String: Any]) throws -> String { @@ -465,12 +556,19 @@ final class MongoScriptHost { var deleted = 0 var upserted = 0 + let concern = try writeConcern(for: request) + let acknowledged = MongoScriptCommandBuilder.isAcknowledged(writeConcern: concern) let statements = try operations.map { operation in - try MongoScriptCommandBuilder.bulkOperation(operation, collection: collectionName(request)) + try MongoScriptCommandBuilder.bulkOperation( + operation, collection: collectionName(request), writeConcern: concern + ) } for statement in statements { - let reply = try writeCommand(statement.document, request) + let reply = try write( + statement.kind, touchesMany: statement.touchesMany, acknowledged: acknowledged, + command: statement.document, request: request + ) switch statement.kind { case .insert: inserted += Int(MongoScriptJson.number(in: reply, key: "n") ?? 0) case .update: @@ -478,9 +576,11 @@ final class MongoScriptHost { modified += Int(MongoScriptJson.number(in: reply, key: "nModified") ?? 0) upserted += MongoScriptJson.member(of: reply, key: "upserted") == nil ? 0 : 1 case .delete: deleted += Int(MongoScriptJson.number(in: reply, key: "n") ?? 0) + case .findAndModify: break } } + guard acknowledged else { return Self.unacknowledgedReply } return """ {"insertedCount": \(inserted), "matchedCount": \(matched), "modifiedCount": \(modified), \ "deletedCount": \(deleted), "upsertedCount": \(upserted)} diff --git a/Plugins/MongoDBDriverPlugin/MongoScriptJson.swift b/Plugins/MongoDBDriverPlugin/MongoScriptJson.swift index 781a983114..54266a7dc8 100644 --- a/Plugins/MongoDBDriverPlugin/MongoScriptJson.swift +++ b/Plugins/MongoDBDriverPlugin/MongoScriptJson.swift @@ -16,6 +16,14 @@ enum MongoScriptJson { "{\"ok\":false,\"e\":{\"m\":\(jsonString(message)),\"c\":\(code)}}" } + /// A failed write, with its stage, so the exception a script lets escape still says it was a + /// write's failure and how far the write got. + static func failure(_ write: MongoWriteFailure) -> String { + """ + {"ok":false,"e":{"m":\(jsonString(write.message)),"c":\(write.code),"s":\(jsonString(write.stage.rawValue))}} + """ + } + static func jsonString(_ value: String) -> String { var escaped = "" escaped.reserveCapacity(value.count + 2) diff --git a/Plugins/MongoDBDriverPlugin/MongoScriptPrelude.swift b/Plugins/MongoDBDriverPlugin/MongoScriptPrelude.swift index 3fe02d173c..e83759ba24 100644 --- a/Plugins/MongoDBDriverPlugin/MongoScriptPrelude.swift +++ b/Plugins/MongoDBDriverPlugin/MongoScriptPrelude.swift @@ -24,6 +24,7 @@ enum MongoScriptPrelude { var failure = new Error(response.e.m); failure.code = response.e.c; failure.isMongoError = true; + if (response.e.s) { failure.__writeStage = response.e.s; } throw failure; } return response.v; @@ -345,25 +346,33 @@ enum MongoScriptPrelude { filter: __ejson(filter === undefined ? {} : filter) }); }; - DBCollection.prototype.insertOne = function (document) { + DBCollection.prototype.insertOne = function (document, options) { if (document === undefined || document === null) { throw new Error("insertOne needs a document"); } - var reply = this.__reply("insertOne", { document: __ejson(document) }); - return { acknowledged: true, insertedId: reply.insertedIds[0] }; + var reply = this.__reply("insertOne", { + document: __ejson(document), + options: options === undefined ? null : __ejson(options) + }); + return { acknowledged: reply.acknowledged !== false, insertedId: reply.insertedIds[0] }; }; - DBCollection.prototype.insertMany = function (documents) { - var reply = this.__reply("insertMany", { documents: __ejson(documents) }); + DBCollection.prototype.insertMany = function (documents, options) { + var reply = this.__reply("insertMany", { + documents: __ejson(documents), + options: options === undefined ? null : __ejson(options) + }); + if (reply.acknowledged === false) { return { acknowledged: false, insertedIds: reply.insertedIds }; } return { acknowledged: true, insertedIds: reply.insertedIds, insertedCount: reply.insertedCount }; }; - DBCollection.prototype.insert = function (documentOrArray) { + DBCollection.prototype.insert = function (documentOrArray, options) { return Array.isArray(documentOrArray) - ? this.insertMany(documentOrArray) - : this.insertOne(documentOrArray); + ? this.insertMany(documentOrArray, options) + : this.insertOne(documentOrArray, options); }; function __updateResult(reply) { + if (reply.acknowledged === false) { return { acknowledged: false }; } // An upsert replies with n = 1 and an `upserted` entry even though nothing matched, so the // upserted rows come out of `n` to give mongosh's matchedCount. var upserted = reply.upserted || []; @@ -412,6 +421,7 @@ enum MongoScriptPrelude { options: options === undefined ? null : __ejson(options), multi: multi }); + if (reply.acknowledged === false) { return { acknowledged: false }; } return { acknowledged: true, deletedCount: reply.n || 0 }; }; DBCollection.prototype.deleteOne = function (filter, options) { @@ -420,8 +430,12 @@ enum MongoScriptPrelude { DBCollection.prototype.deleteMany = function (filter, options) { return this.__delete(filter, options, true); }; - DBCollection.prototype.remove = function (filter, justOne) { - return justOne === true ? this.deleteOne(filter) : this.deleteMany(filter); + DBCollection.prototype.remove = function (filter, justOneOrOptions) { + if (typeof justOneOrOptions === "boolean") { + return justOneOrOptions ? this.deleteOne(filter) : this.deleteMany(filter); + } + var options = justOneOrOptions === null ? undefined : justOneOrOptions; + return options && options.justOne ? this.deleteOne(filter, options) : this.deleteMany(filter, options); }; DBCollection.prototype.__findAndModify = function (filter, change, options, remove) { if (!remove && (change === undefined || change === null)) { @@ -444,8 +458,12 @@ enum MongoScriptPrelude { DBCollection.prototype.findOneAndDelete = function (filter, options) { return this.__findAndModify(filter, undefined, options, true); }; - DBCollection.prototype.bulkWrite = function (operations) { - var reply = this.__reply("bulkWrite", { operations: __ejson(operations) }); + DBCollection.prototype.bulkWrite = function (operations, options) { + var reply = this.__reply("bulkWrite", { + operations: __ejson(operations), + options: options === undefined ? null : __ejson(options) + }); + if (reply.acknowledged === false) { return { acknowledged: false }; } reply.acknowledged = true; return reply; }; diff --git a/Plugins/MongoDBDriverPlugin/MongoScriptRuntime.swift b/Plugins/MongoDBDriverPlugin/MongoScriptRuntime.swift index 95c327bf16..7febbc4e23 100644 --- a/Plugins/MongoDBDriverPlugin/MongoScriptRuntime.swift +++ b/Plugins/MongoDBDriverPlugin/MongoScriptRuntime.swift @@ -84,7 +84,9 @@ final class MongoScriptRuntime: @unchecked Sendable { do { return try self.run(statement, on: engine) } catch { - throw MongoScriptStatementFailure.carrying(error, databaseSwitch: engine.host.databaseSwitch) + throw MongoScriptStatementFailure.carrying( + error, databaseSwitch: engine.host.databaseSwitch, writes: engine.host.writes + ) } }) } @@ -106,9 +108,12 @@ final class MongoScriptRuntime: @unchecked Sendable { watchForSilence(engine: engine, gate: gate) return } - guard gate.finish(with: .failure(MongoDBError( - code: 0, message: MongoScriptText.timedOut(Self.silenceLimit) - ))) else { return } + let abandoned = MongoScriptStatementFailure.carrying( + MongoDBError(code: 0, message: MongoScriptText.timedOut(Self.silenceLimit)), + databaseSwitch: nil, + writes: engine.host.writes + ) + guard gate.finish(with: .failure(abandoned)) else { return } poison(engine) } } @@ -149,7 +154,9 @@ final class MongoScriptRuntime: @unchecked Sendable { do { return try self.runForExport(statement, on: engine) } catch { - throw MongoScriptStatementFailure.carrying(error, databaseSwitch: engine.host.databaseSwitch) + throw MongoScriptStatementFailure.carrying( + error, databaseSwitch: engine.host.databaseSwitch, writes: engine.host.writes + ) } }) } @@ -171,7 +178,7 @@ final class MongoScriptRuntime: @unchecked Sendable { if let value, value.isObject, let handle = value.objectForKeyedSubscript("__handle"), handle.isNumber, let plan = engine.host.cursorPlan(handle: Int(handle.toInt32())) { - return .cursor(plan) + return .cursor(plan, databaseSwitch: engine.host.databaseSwitch, writes: engine.host.writes) } var result = MongoScriptStatementResult() @@ -340,6 +347,7 @@ final class MongoScriptRuntime: @unchecked Sendable { /// The shell answers a write with the object mongosh answers with, so the count the result bar /// reports has to be read back out of it rather than counted from the grid's one row. private static func rowsAffected(in json: String) -> Int { + guard MongoScriptJson.member(of: json, key: "acknowledged") != "false" else { return 0 } // Summed, not first-wins: a `bulkWrite` that mixes an insert with a delete carries two // positive counters and affected both rows. let total = ["modifiedCount", "deletedCount", "insertedCount", "upsertedCount"] @@ -354,6 +362,7 @@ final class MongoScriptRuntime: @unchecked Sendable { if exception.objectForKeyedSubscript("isMongoError")?.toBool() == true { let message = exception.objectForKeyedSubscript("message")?.toString() ?? "" if message == MongoScriptText.cancelled { return CancellationError() } + if let failedWrite = MongoScriptContext.writeFailure(in: exception) { return failedWrite } let code = UInt32(max(0, exception.objectForKeyedSubscript("code")?.toInt32() ?? 0)) return MongoDBError(code: code, message: message) } diff --git a/Plugins/MongoDBDriverPlugin/MongoScriptText.swift b/Plugins/MongoDBDriverPlugin/MongoScriptText.swift index 1aaac4658a..9ef58e702d 100644 --- a/Plugins/MongoDBDriverPlugin/MongoScriptText.swift +++ b/Plugins/MongoDBDriverPlugin/MongoScriptText.swift @@ -81,10 +81,47 @@ enum MongoScriptText { String(format: String(localized: "MongoDB refused the write (error %u)."), code) } + static var insertStoppedAtOversizedDocument: String { + String(localized: "The insert stopped at a document larger than MongoDB accepts.") + } + static func writeNotAcknowledged(reason: String) -> String { String( format: String(localized: "The write was applied, but the servers did not confirm it as the write concern asks: %@"), reason ) } + + static func writesChanged(_ count: Int) -> String { + String( + format: String( + localized: """ + %d document(s) had already been changed when this failed. \ + MongoDB does not undo them, so check the data before you run it again. + """ + ), + count + ) + } + + static var writesMayHaveChanged: String { + String( + localized: """ + Some documents may already have been changed when this failed. \ + MongoDB does not undo them, so check the data before you run it again. + """ + ) + } + + static func writesChangedAndMaybeMore(_ count: Int) -> String { + String( + format: String( + localized: """ + %d document(s), and possibly more that MongoDB does not report, had already been changed \ + when this failed. MongoDB does not undo them, so check the data before you run it again. + """ + ), + count + ) + } } diff --git a/Plugins/MongoDBDriverPlugin/MongoWriteFailure.swift b/Plugins/MongoDBDriverPlugin/MongoWriteFailure.swift index 45f3f3c8ef..bc218f2c3b 100644 --- a/Plugins/MongoDBDriverPlugin/MongoWriteFailure.swift +++ b/Plugins/MongoDBDriverPlugin/MongoWriteFailure.swift @@ -1,6 +1,6 @@ import Foundation -/// What a write command's reply says went wrong when the command itself ran. +/// What a write command's reply says went wrong. /// /// `mongoc_client_command_simple` answers only whether the server ran the command. An `update`, /// `delete`, `insert` or `findAndModify` it ran can still refuse the documents it reached, and the @@ -10,33 +10,76 @@ import Foundation /// /// A write-concern error arrives after the write itself was applied, so its message says so rather /// than reading like a refusal. -struct MongoWriteFailure: Equatable, Sendable { +/// +/// The stage says how far the write got, which is what decides whether documents may already have +/// changed. A command the server stopped while it ran (`ok: 0` with a code in the `Interruption` +/// category) keeps what it had written, and the reply does not say how much. That test reads the +/// code from the reply, never from libmongoc's error, whose own `24` and `50` mean something else. +/// +/// It is also the error the host throws for a failed write, and the stage crosses into the script +/// with it. That is how the driver tells a write's failure from a read's when both carry the same +/// code and message: a script can catch one and go on to fail on the other. +struct MongoWriteFailure: Error, LocalizedError, Equatable, Sendable { + enum Stage: String, Equatable, Sendable { + case document + case unconfirmed + case command + case unanswered + case notSent + /// An insert that sent some batches and then met a document it could not send. + case stoppedBetweenBatches + } + let code: UInt32 let message: String + let stage: Stage + + var errorDescription: String? { message } + var stoppedWhileRunning: Bool { + stage == .command && MongoDBServerErrorCode.interruptionCategory.contains(code) + } + + /// Reads the raw command reply, and the reply the CRUD calls build, which carries + /// `writeConcernErrors` and `errorReplies` as arrays and has no `ok` of its own. static func read(fromReply replyJson: String) -> MongoWriteFailure? { guard let data = replyJson.data(using: .utf8), let reply = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return nil } if let writeErrors = reply["writeErrors"] as? [[String: Any]], let first = writeErrors.first { - return entry(first) + return entry(first, stage: .document) } - if let concernError = reply["writeConcernError"] as? [String: Any] { - let failure = entry(concernError) + if let concernError = reply["writeConcernError"] as? [String: Any] + ?? (reply["writeConcernErrors"] as? [[String: Any]])?.first { + let failure = entry(concernError, stage: .unconfirmed) return MongoWriteFailure( code: failure.code, - message: MongoScriptText.writeNotAcknowledged(reason: failure.message) + message: MongoScriptText.writeNotAcknowledged(reason: failure.message), + stage: .unconfirmed ) } + if MongoScriptJson.numeric(reply["ok"]) == 0 { + return entry(reply, stage: .command) + } + if let errorReply = (reply["errorReplies"] as? [[String: Any]])?.first { + return entry(errorReply, stage: .command) + } return nil } - private static func entry(_ entry: [String: Any]) -> MongoWriteFailure { + private static func entry(_ entry: [String: Any], stage: Stage) -> MongoWriteFailure { let code = UInt32(clamping: MongoScriptJson.numeric(entry["code"]) ?? 0) guard let message = entry["errmsg"] as? String, !message.isEmpty else { - return MongoWriteFailure(code: code, message: MongoScriptText.writeRefused(code: code)) + return MongoWriteFailure(code: code, message: MongoScriptText.writeRefused(code: code), stage: stage) } - return MongoWriteFailure(code: code, message: message) + return MongoWriteFailure(code: code, message: message, stage: stage) } } + +/// One write's reply, kept whether or not the write failed, because a failed write can still have +/// changed documents and the reply is the only record of how many. +struct MongoWriteOutcome: Sendable { + let replyJson: String + let failure: MongoWriteFailure? +} diff --git a/Plugins/MongoDBDriverPlugin/MongoWriteLedger.swift b/Plugins/MongoDBDriverPlugin/MongoWriteLedger.swift new file mode 100644 index 0000000000..a47b3cac41 --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoWriteLedger.swift @@ -0,0 +1,131 @@ +import Foundation + +enum MongoWriteOperation: Equatable, Sendable { + case insert + case update + case delete + case findAndModify +} + +/// What one statement has already written, so a failure can say so. +/// +/// MongoDB undoes nothing outside a transaction. An `updateMany` that fails on its third document +/// keeps the first two, an ordered `insertMany` keeps every document before the duplicate, and a +/// multi-document write stopped by a timeout keeps whatever it reached. The error alone reads as +/// if nothing happened, and running the statement again applies the change twice. +/// +/// The count comes from each reply, and where a reply cannot tell, the ledger says documents may +/// have changed rather than guessing a number: an `updateMany` that fails part-way replies +/// `nModified: 0` however many it changed first. A write sent with `w: 0` cannot tell either: the +/// server answers it with `n: 0` whatever it changed, and says nothing of a write it refused. +/// +/// A failure that stopped a write before it was sent still counts what the reply says was written +/// first. libmongoc sends a large `insertMany` in batches, and one that stops at a document it +/// cannot send has already inserted the batches before it. +struct MongoWriteLedger: Equatable, Sendable { + private(set) var changed = 0 + private(set) var mayHaveChangedMore = false + private(set) var isWriting = false + + var isEmpty: Bool { self == MongoWriteLedger() } + + mutating func beginWrite() { + isWriting = true + } + + mutating func abandonWrite() { + isWriting = false + } + + mutating func record( + _ operation: MongoWriteOperation, + touchesMany: Bool, + acknowledged: Bool, + outcome: MongoWriteOutcome + ) { + isWriting = false + let applied = Self.appliedCount(operation, reply: outcome.replyJson) + guard let failure = outcome.failure else { + guard acknowledged else { + mayHaveChangedMore = true + return + } + changed += applied + return + } + changed += applied + switch failure.stage { + case .unconfirmed, .notSent: + break + case .stoppedBetweenBatches: + if !acknowledged { mayHaveChangedMore = true } + case .document: + if operation != .insert, touchesMany { mayHaveChangedMore = true } + case .command: + if failure.stoppedWhileRunning, touchesMany { mayHaveChangedMore = true } + case .unanswered: + mayHaveChangedMore = true + } + } + + /// A write still waiting on the server when the statement was abandoned may have been applied. + var note: String? { + let unknown = mayHaveChangedMore || isWriting + switch (changed, unknown) { + case (0, false): return nil + case (0, true): return MongoScriptText.writesMayHaveChanged + case (let count, false): return MongoScriptText.writesChanged(count) + case (let count, true): return MongoScriptText.writesChangedAndMaybeMore(count) + } + } + + /// The message a failed statement reports: the timeout wording first, the note after it. + /// + /// `failedWrite` is the stage of the write whose failure escaped the statement, and nil when + /// what escaped was not a write's failure. Only a server timeout is reworded, as a write when a + /// write's failure escaped and as a query otherwise. A write-concern error keeps its own text + /// even when its code is the timeout's, because that text is the one that says the write was + /// applied. + func reportedMessage( + code: UInt32, + message: String, + failedWrite: MongoWriteFailure.Stage?, + maxTimeMS: Int32? + ) -> String { + let base = timeoutMessage(code: code, failedWrite: failedWrite, maxTimeMS: maxTimeMS) ?? message + guard let note else { return base } + return "\(base)\n\n\(note)" + } + + private func timeoutMessage(code: UInt32, failedWrite: MongoWriteFailure.Stage?, maxTimeMS: Int32?) -> String? { + guard MongoDBTimeoutPolicy.isTimeoutCode(code), let maxTimeMS else { return nil } + switch failedWrite { + case nil: + return MongoDBTimeoutPolicy.timeoutMessage(maxTimeMS: maxTimeMS) + case .unconfirmed: + return nil + case .document, .command, .unanswered, .notSent, .stoppedBetweenBatches: + return MongoDBTimeoutPolicy.writeTimeoutMessage(maxTimeMS: maxTimeMS) + } + } + + private static func appliedCount(_ operation: MongoWriteOperation, reply: String) -> Int { + switch operation { + case .insert: + return count(in: reply, key: "insertedCount") ?? count(in: reply, key: "n") ?? 0 + case .update: + let upserted = MongoScriptJson.member(of: reply, key: "upserted") + .map { MongoScriptJson.topLevelElements($0).count } ?? 0 + return (count(in: reply, key: "nModified") ?? 0) + upserted + case .delete: + return count(in: reply, key: "n") ?? 0 + case .findAndModify: + return MongoScriptJson.member(of: reply, key: "lastErrorObject") + .flatMap { count(in: $0, key: "n") } ?? 0 + } + } + + private static func count(in json: String, key: String) -> Int? { + MongoScriptJson.number(in: json, key: key).map { Int(clamping: max(0, $0)) } + } +} diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 55d3752f85..9ccbb1a320 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -184363,6 +184363,21 @@ }, "This database does not store whole documents" : { + }, + "The write did not finish within %d seconds, so MongoDB stopped it. Raise the query timeout in Settings if it needs longer." : { + + }, + "%d document(s) had already been changed when this failed. MongoDB does not undo them, so check the data before you run it again." : { + + }, + "Some documents may already have been changed when this failed. MongoDB does not undo them, so check the data before you run it again." : { + + }, + "%d document(s), and possibly more that MongoDB does not report, had already been changed when this failed. MongoDB does not undo them, so check the data before you run it again." : { + + }, + "The insert stopped at a document larger than MongoDB accepts." : { + } }, "version" : "1.1" diff --git a/TableProTests/Core/MongoDB/MongoDBTimeoutPolicyTests.swift b/TableProTests/Core/MongoDB/MongoDBTimeoutPolicyTests.swift index 65580a319e..41dad97c07 100644 --- a/TableProTests/Core/MongoDB/MongoDBTimeoutPolicyTests.swift +++ b/TableProTests/Core/MongoDB/MongoDBTimeoutPolicyTests.swift @@ -81,6 +81,51 @@ struct MongoDBTimeoutPolicyTests { } } + @Test("A write that timed out says it was stopped and names the seconds, rounded to at least one") + func writeTimeoutMessage() { + let message = MongoDBTimeoutPolicy.writeTimeoutMessage(maxTimeMS: 30_000) + #expect(message.contains("30")) + #expect(message.localizedCaseInsensitiveContains("write")) + #expect(!message.localizedCaseInsensitiveContains("index")) + #expect(MongoDBTimeoutPolicy.writeTimeoutMessage(maxTimeMS: 200) + == MongoDBTimeoutPolicy.writeTimeoutMessage(maxTimeMS: 1_000)) + #expect(MongoDBTimeoutPolicy.writeTimeoutMessage(maxTimeMS: 1_600) + == MongoDBTimeoutPolicy.writeTimeoutMessage(maxTimeMS: 2_000)) + } + + @Test("The interruption category holds the timeout and a kill, and not a refusal") + func interruptionCategory() { + let category = MongoDBServerErrorCode.interruptionCategory + #expect(category.contains(MongoDBServerErrorCode.maxTimeMSExpired)) + #expect(category.contains(11_601)) + #expect(category.contains(11_602)) + #expect(!category.contains(MongoDBServerErrorCode.badValue)) + #expect(!category.contains(121)) + #expect(!category.contains(11_000)) + } + + @Test("The interruption category holds every Interruption code from MongoDB 4.0 through 9.0") + func interruptionCategoryCoversEveryRelease() { + let addedByRelease: [(release: String, codes: [UInt32])] = [ + ("4.0", [24, 50, 237, 262, 11_600, 11_601, 11_602]), + ("4.2", [279, 282, 46_841]), + ("4.4", [281, 290]), + ("5.1", [355]), + ("8.0", [91_331]), + ("8.1", [10_045_600]), + ("8.2", [453]), + ("8.3", [471, 473, 485]), + ("9.0", [509]) + ] + let category = MongoDBServerErrorCode.interruptionCategory + for (release, codes) in addedByRelease { + for code in codes { + #expect(category.contains(code), "MongoDB \(release) code \(code)") + } + } + #expect(category.count == addedByRelease.flatMap { $0.codes }.count) + } + @Test("Only MaxTimeMSExpired counts as a timeout") func timeoutCodeDetection() { #expect(MongoDBTimeoutPolicy.isTimeoutCode(50)) diff --git a/TableProTests/Core/MongoDB/MongoScriptCommandTests.swift b/TableProTests/Core/MongoDB/MongoScriptCommandTests.swift index 6fcd19fb9d..c0fb2604ec 100644 --- a/TableProTests/Core/MongoDB/MongoScriptCommandTests.swift +++ b/TableProTests/Core/MongoDB/MongoScriptCommandTests.swift @@ -168,7 +168,7 @@ struct MongoScriptCommandBuilderTests { func updateMany() { let command = MongoScriptCommandBuilder.update( collection: "orders", filter: "{\"a\":1}", update: "{\"$set\":{\"b\":2}}", - multi: true, options: [:] + multi: true, options: [:], writeConcern: nil ) #expect(command.contains("\"update\": \"orders\"")) #expect(command.contains("\"q\": {\"a\":1}")) @@ -181,7 +181,7 @@ struct MongoScriptCommandBuilderTests { func updateOptions() { let command = MongoScriptCommandBuilder.update( collection: "orders", filter: "{}", update: "{}", multi: false, - options: ["upsert": true, "arrayFilters": [["x.y": 1]]] + options: ["upsert": true, "arrayFilters": [["x.y": 1]]], writeConcern: nil ) #expect(command.contains("\"upsert\": true")) #expect(command.contains("\"arrayFilters\":")) @@ -191,12 +191,12 @@ struct MongoScriptCommandBuilderTests { func deleteLimits() { #expect( MongoScriptCommandBuilder - .delete(collection: "orders", filter: "{}", multi: false, options: [:]) + .delete(collection: "orders", filter: "{}", multi: false, options: [:], writeConcern: nil) .contains("\"limit\": 1") ) #expect( MongoScriptCommandBuilder - .delete(collection: "orders", filter: "{}", multi: true, options: [:]) + .delete(collection: "orders", filter: "{}", multi: true, options: [:], writeConcern: nil) .contains("\"limit\": 0") ) } @@ -204,7 +204,7 @@ struct MongoScriptCommandBuilderTests { @Test("findOneAndDelete asks the server to remove rather than update") func findAndModifyRemove() { let command = MongoScriptCommandBuilder.findAndModify( - collection: "orders", filter: "{\"a\":1}", update: nil, remove: true, options: [:] + collection: "orders", filter: "{\"a\":1}", update: nil, remove: true, options: [:], writeConcern: nil ) #expect(command.contains("\"remove\": true")) #expect(!command.contains("\"update\"")) @@ -214,7 +214,7 @@ struct MongoScriptCommandBuilderTests { func findAndModifyReturnsNew() { let command = MongoScriptCommandBuilder.findAndModify( collection: "orders", filter: "{}", update: "{\"$set\":{}}", remove: false, - options: ["returnDocument": "after"] + options: ["returnDocument": "after"], writeConcern: nil ) #expect(command.contains("\"new\": true")) } @@ -257,31 +257,243 @@ struct MongoScriptCommandBuilderTests { @Test("bulkWrite maps each operation to its own command") func bulkOperations() throws { let insert = try MongoScriptCommandBuilder.bulkOperation( - "{\"insertOne\": {\"document\": {\"a\": 1}}}", collection: "orders" + "{\"insertOne\": {\"document\": {\"a\": 1}}}", collection: "orders", writeConcern: nil ) #expect(insert.kind == .insert) #expect(insert.document.contains("\"documents\": [{\"a\": 1}]")) let update = try MongoScriptCommandBuilder.bulkOperation( "{\"updateMany\": {\"filter\": {\"a\": 1}, \"update\": {\"$set\": {\"b\": 2}}}}", - collection: "orders" + collection: "orders", writeConcern: nil ) #expect(update.kind == .update) + #expect(update.touchesMany) #expect(update.document.contains("\"multi\": true")) let delete = try MongoScriptCommandBuilder.bulkOperation( - "{\"deleteOne\": {\"filter\": {\"a\": 1}}}", collection: "orders" + "{\"deleteOne\": {\"filter\": {\"a\": 1}}}", collection: "orders", writeConcern: nil ) #expect(delete.kind == .delete) + #expect(!delete.touchesMany) #expect(delete.document.contains("\"limit\": 1")) } + @Test("A bulk delete keeps its collation instead of dropping it") + func bulkDeleteOptions() throws { + let delete = try MongoScriptCommandBuilder.bulkOperation( + "{\"deleteMany\": {\"filter\": {\"a\": 1}, \"collation\": {\"locale\": \"fr\"}}}", + collection: "orders", writeConcern: nil + ) + #expect(delete.touchesMany) + #expect(delete.document.contains("\"collation\": {\"locale\":\"fr\"}")) + } + @Test("An unknown bulk operation is refused rather than silently skipped") func unknownBulkOperation() { #expect(throws: MongoScriptError.self) { - try MongoScriptCommandBuilder.bulkOperation("{\"upsertAll\": {}}", collection: "orders") + try MongoScriptCommandBuilder.bulkOperation("{\"upsertAll\": {}}", collection: "orders", writeConcern: nil) + } + } +} + +struct MongoScriptWriteConcernTests { + private static let majority = "{\"w\": \"majority\"}" + + private static func occurrences(of field: String, in command: String) -> Int { + command.components(separatedBy: field).count - 1 + } + + @Test("A statement that names no write concern takes the connection's") + func connectionDefault() { + #expect(MongoScriptCommandBuilder.writeConcern(statementOptions: nil, connectionDefault: Self.majority) == Self.majority) + #expect( + MongoScriptCommandBuilder.writeConcern(statementOptions: "{\"upsert\":true}", connectionDefault: Self.majority) + == Self.majority + ) + } + + @Test("The statement's own write concern wins over the connection's, as it does in mongosh") + func statementWins() { + let options = "{\"writeConcern\":{\"w\":{\"$numberInt\":\"1\"},\"wtimeout\":{\"$numberInt\":\"777\"}}}" + #expect( + MongoScriptCommandBuilder.writeConcern(statementOptions: options, connectionDefault: Self.majority) + == "{\"w\": {\"$numberInt\":\"1\"}, \"wtimeout\": {\"$numberInt\":\"777\"}}" + ) + } + + @Test("mongosh's journal and wtimeoutMS go out as the j and wtimeout the server reads") + func mongoshSpellings() { + let options = "{\"writeConcern\":{\"journal\":true,\"wtimeoutMS\":{\"$numberInt\":\"1000\"}}}" + let canonical = "{\"j\": true, \"wtimeout\": {\"$numberInt\":\"1000\"}}" + #expect( + MongoScriptCommandBuilder.writeConcern(statementOptions: options, connectionDefault: Self.majority) + == canonical + ) + #expect(MongoScriptCommandBuilder.insertOptions(statementOptions: options) == "{\"writeConcern\": \(canonical)}") + } + + @Test("Each name takes the first spelling set, in mongosh's order, and fsync stands for j") + func spellingPrecedence() { + let cases = [ + ("{\"writeConcern\":{\"journal\":true,\"j\":false}}", "{\"j\": false}"), + ("{\"writeConcern\":{\"fsync\":true,\"w\":{\"$numberInt\":\"1\"}}}", "{\"w\": {\"$numberInt\":\"1\"}, \"j\": true}"), + ("{\"writeConcern\":{\"wtimeoutMS\":9,\"wtimeout\":5}}", "{\"wtimeout\": 5}"), + ("{\"writeConcern\":{\"j\":null,\"journal\":true}}", "{\"j\": true}") + ] + for (options, expected) in cases { + #expect(MongoScriptCommandBuilder.writeConcern(statementOptions: options, connectionDefault: nil) == expected) + } + } + + @Test("A write concern naming none of w, j and wtimeout falls back to the connection's, as in mongosh") + func emptyFallsBack() { + for options in ["{\"writeConcern\":{}}", "{\"writeConcern\":{\"foo\":1}}", "{\"writeConcern\":{\"w\":null}}"] { + #expect( + MongoScriptCommandBuilder.writeConcern(statementOptions: options, connectionDefault: Self.majority) + == Self.majority + ) + #expect(MongoScriptCommandBuilder.insertOptions(statementOptions: options) == nil) + } + } + + @Test("A write concern that names one field replaces the connection's whole, as in mongosh") + func partialReplacesWhole() { + #expect( + MongoScriptCommandBuilder.writeConcern( + statementOptions: "{\"writeConcern\":{\"wtimeoutMS\":{\"$numberInt\":\"555\"}}}", + connectionDefault: Self.majority + ) == "{\"wtimeout\": {\"$numberInt\":\"555\"}}" + ) + } + + @Test("Only w: 0, or libmongoc's legacy w: -1, without j: true goes unacknowledged, in any number wrapper") + func acknowledgement() { + let unacknowledged = [ + "{\"w\": {\"$numberInt\":\"0\"}}", + "{\"w\": {\"$numberDouble\":\"0.0\"}}", + "{\"w\": {\"$numberInt\":\"0\"}, \"j\": false}", + "{\"w\":{\"$numberInt\":\"0\"}}", + "{\"w\": {\"$numberInt\":\"-1\"}}", + "{\"w\": -1}" + ] + for concern in unacknowledged { + #expect(!MongoScriptCommandBuilder.isAcknowledged(writeConcern: concern), "\(concern)") + } + let acknowledged = [ + nil, + Self.majority, + "{\"w\": {\"$numberInt\":\"1\"}}", + "{\"w\": {\"$numberInt\":\"0\"}, \"j\": true}", + "{\"w\": \"0\"}", + "{\"w\": false}", + "{\"j\": true}" + ] + for concern in acknowledged { + #expect(MongoScriptCommandBuilder.isAcknowledged(writeConcern: concern), "\(String(describing: concern))") + } + } + + @Test("A statement's w: 0 decides acknowledgement the same way the connection's does") + func statementWZero() { + let concern = MongoScriptCommandBuilder.writeConcern( + statementOptions: "{\"writeConcern\":{\"w\":{\"$numberInt\":\"0\"}}}", connectionDefault: Self.majority + ) + #expect(!MongoScriptCommandBuilder.isAcknowledged(writeConcern: concern)) + #expect(!MongoScriptCommandBuilder.isAcknowledged( + writeConcern: MongoScriptCommandBuilder.writeConcern( + statementOptions: nil, connectionDefault: "{ \"w\" : { \"$numberInt\" : \"0\" } }" + ) + )) + } + + @Test("A null write concern counts as unset, the way mongosh reads it") + func nullIsUnset() { + let concern = MongoScriptCommandBuilder.writeConcern( + statementOptions: "{\"writeConcern\":null}", connectionDefault: Self.majority + ) + #expect(concern == Self.majority) + let command = MongoScriptCommandBuilder.update( + collection: "orders", filter: "{}", update: "{\"$set\":{}}", multi: false, + options: [:], writeConcern: concern + ) + #expect(Self.occurrences(of: "\"writeConcern\"", in: command) == 1) + #expect(command.contains("\"writeConcern\": \(Self.majority)")) + } + + @Test("With neither set, no write concern is sent and the server's default applies") + func neitherSet() { + #expect(MongoScriptCommandBuilder.writeConcern(statementOptions: nil, connectionDefault: nil) == nil) + let command = MongoScriptCommandBuilder.delete( + collection: "orders", filter: "{}", multi: true, options: [:], writeConcern: nil + ) + #expect(!command.contains("writeConcern")) + } + + @Test("Every write command carries the write concern exactly once, at the top level") + func everyWriteCarriesIt() throws { + let commands = [ + MongoScriptCommandBuilder.update( + collection: "orders", filter: "{}", update: "{\"$set\":{}}", multi: true, + options: [:], writeConcern: Self.majority + ), + MongoScriptCommandBuilder.delete( + collection: "orders", filter: "{}", multi: false, options: [:], writeConcern: Self.majority + ), + MongoScriptCommandBuilder.findAndModify( + collection: "orders", filter: "{}", update: "{\"$set\":{}}", remove: false, + options: [:], writeConcern: Self.majority + ), + try MongoScriptCommandBuilder.bulkOperation( + "{\"insertOne\": {\"document\": {\"a\": 1}}}", collection: "orders", writeConcern: Self.majority + ).document, + try MongoScriptCommandBuilder.bulkOperation( + "{\"updateMany\": {\"filter\": {}, \"update\": {\"$set\": {\"b\": 2}}}}", + collection: "orders", writeConcern: Self.majority + ).document, + try MongoScriptCommandBuilder.bulkOperation( + "{\"deleteOne\": {\"filter\": {}}}", collection: "orders", writeConcern: Self.majority + ).document + ] + for command in commands { + #expect(Self.occurrences(of: "\"writeConcern\"", in: command) == 1) + #expect(MongoScriptJson.member(of: command, key: "writeConcern") == Self.majority) } } + + @Test("An insert sends w: 0 beside j: true as w: 1, which libmongoc accepts and the server treats the same") + func insertJournaledUnacknowledged() { + let cases = [ + ("{\"writeConcern\":{\"w\":{\"$numberInt\":\"0\"},\"j\":true}}", "{\"w\": 1, \"j\": true}"), + ("{\"writeConcern\":{\"w\":0,\"journal\":true,\"wtimeoutMS\":5}}", "{\"w\": 1, \"j\": true, \"wtimeout\": 5}"), + ("{\"writeConcern\":{\"fsync\":true,\"w\":0}}", "{\"w\": 1, \"j\": true}"), + ("{\"writeConcern\":{\"w\":0,\"j\":false}}", "{\"w\": 0, \"j\": false}"), + ("{\"writeConcern\":{\"w\":-1,\"j\":true}}", "{\"w\": -1, \"j\": true}") + ] + for (options, concern) in cases { + #expect(MongoScriptCommandBuilder.insertOptions(statementOptions: options) == "{\"writeConcern\": \(concern)}") + } + let commandConcern = MongoScriptCommandBuilder.writeConcern( + statementOptions: "{\"writeConcern\":{\"w\":0,\"j\":true}}", connectionDefault: Self.majority + ) + #expect(commandConcern == "{\"w\": 0, \"j\": true}") + #expect(MongoScriptCommandBuilder.isAcknowledged(writeConcern: commandConcern)) + } + + @Test("An insert takes only the statement's own write concern and ordered flag") + func insertOptions() { + #expect(MongoScriptCommandBuilder.insertOptions(statementOptions: nil) == nil) + #expect(MongoScriptCommandBuilder.insertOptions(statementOptions: "null") == nil) + #expect(MongoScriptCommandBuilder.insertOptions(statementOptions: "{\"comment\":\"x\"}") == nil) + #expect( + MongoScriptCommandBuilder.insertOptions( + statementOptions: "{\"ordered\":false,\"comment\":\"x\",\"writeConcern\":{\"w\":\"majority\"}}" + ) == "{\"writeConcern\": {\"w\": \"majority\"}, \"ordered\": false}" + ) + #expect( + MongoScriptCommandBuilder.insertOptions(statementOptions: "{\"writeConcern\":null,\"ordered\":true}") + == "{\"ordered\": true}" + ) + } } struct MongoScriptObjectIdTests { @@ -294,7 +506,9 @@ struct MongoScriptObjectIdTests { @Test("Two generated ids differ") func uniqueness() { - #expect(MongoScriptObjectId.generate() != MongoScriptObjectId.generate()) + let first = MongoScriptObjectId.generate() + let second = MongoScriptObjectId.generate() + #expect(first != second) } @Test("The leading four bytes are the current time") diff --git a/TableProTests/Core/MongoDB/MongoScriptPreludeTests.swift b/TableProTests/Core/MongoDB/MongoScriptPreludeTests.swift index 7026a57126..dabe12b552 100644 --- a/TableProTests/Core/MongoDB/MongoScriptPreludeTests.swift +++ b/TableProTests/Core/MongoDB/MongoScriptPreludeTests.swift @@ -20,6 +20,7 @@ struct MongoScriptPreludeTests { private(set) var printed: [String] = [] var replies: [String] = [] var refusal: (message: String, code: Int)? + var refusals: [String: String] = [:] var database = "shop" func handle(_ requestJson: String) -> String { @@ -39,6 +40,9 @@ struct MongoScriptPreludeTests { case "cursorConfigure", "cursorClose", "useDatabase", "sleep": return "{\"ok\":true,\"v\":null}" default: + if let op = request["op"] as? String, let refused = refusals[op] { + return refused + } if let refusal { return "{\"ok\":false,\"e\":{\"m\":\"\(refusal.message)\",\"c\":\(refusal.code)}}" } @@ -440,6 +444,175 @@ struct MongoScriptPreludeTests { #expect(host.requests(op: "insertOne").isEmpty) } + @Test("insertOne, insertMany and bulkWrite pass their options on, and send null without them") + func writeOptionsReachTheHost() throws { + let host = RecordingHost() + host.replies = [ + "{\"insertedIds\": [1], \"insertedCount\": 1}", + "{\"insertedIds\": [1], \"insertedCount\": 1}", + "{\"insertedIds\": [1], \"insertedCount\": 1}", + "{\"insertedCount\": 0}", + "{\"insertedIds\": [1], \"insertedCount\": 1}", + "{\"insertedCount\": 0}" + ] + let context = try makeContext(host) + + for statement in [ + "db.orders.insertMany([{a: 1}], {ordered: false, writeConcern: {w: 'majority'}})", + "db.orders.insertOne({a: 1}, {writeConcern: {w: 'majority'}})", + "db.orders.insertOne({a: 1})", + "db.orders.bulkWrite([{deleteOne: {filter: {a: 1}}}], {writeConcern: {w: 'majority'}})", + "db.orders.insertMany([{a: 1}])", + "db.orders.bulkWrite([{deleteOne: {filter: {a: 1}}}])" + ] { + context.evaluateScript(statement) + #expect(context.exception == nil, "\(statement) threw") + } + + let insertMany = host.requests(op: "insertMany") + #expect(insertMany.first?["options"] as? String == "{\"ordered\":false,\"writeConcern\":{\"w\":\"majority\"}}") + #expect(insertMany.last?["options"] is NSNull) + let insertOne = host.requests(op: "insertOne") + #expect(insertOne.first?["options"] as? String == "{\"writeConcern\":{\"w\":\"majority\"}}") + #expect(insertOne.last?["options"] is NSNull) + let bulkWrite = host.requests(op: "bulkWrite") + #expect(bulkWrite.first?["options"] as? String == "{\"writeConcern\":{\"w\":\"majority\"}}") + #expect(bulkWrite.last?["options"] is NSNull) + } + + @Test("The legacy insert passes its options to whichever insert it becomes") + func legacyInsertOptions() throws { + let host = RecordingHost() + host.replies = [ + "{\"insertedIds\": [1], \"insertedCount\": 1}", + "{\"insertedIds\": [1, 2], \"insertedCount\": 2}" + ] + let context = try makeContext(host) + + context.evaluateScript("db.orders.insert({a: 1}, {writeConcern: {w: 1}})") + context.evaluateScript("db.orders.insert([{a: 1}, {a: 2}], {ordered: false})") + #expect(context.exception == nil) + + #expect(host.requests(op: "insertOne").first?["options"] as? String + == "{\"writeConcern\":{\"w\":{\"$numberInt\":\"1\"}}}") + #expect(host.requests(op: "insertMany").first?["options"] as? String == "{\"ordered\":false}") + } + + @Test("remove with {justOne: true} deletes one document, and its options reach the host") + func legacyRemoveOptions() throws { + let host = RecordingHost() + host.replies = ["{\"n\": 1}", "{\"n\": 1}", "{\"n\": 3}", "{\"n\": 3}", "{\"n\": 3}"] + let context = try makeContext(host) + + for statement in [ + "db.orders.remove({x: 1}, {justOne: true, writeConcern: {w: 'majority'}})", + "db.orders.remove({x: 1}, true)", + "db.orders.remove({x: 1}, {writeConcern: {w: 'majority'}})", + "db.orders.remove({x: 1}, false)", + "db.orders.remove({x: 1})" + ] { + context.evaluateScript(statement) + #expect(context.exception == nil, "\(statement) threw") + } + + let deletes = host.requests(op: "delete") + try #require(deletes.count == 5) + #expect(deletes.map { $0["multi"] as? Bool } == [false, false, true, true, true]) + #expect(deletes[0]["options"] as? String == "{\"justOne\":true,\"writeConcern\":{\"w\":\"majority\"}}") + #expect(deletes[2]["options"] as? String == "{\"writeConcern\":{\"w\":\"majority\"}}") + #expect(deletes[4]["options"] is NSNull) + } + + @Test("A write sent with w: 0 reports acknowledged false and no counts, as mongosh does") + func unacknowledgedWriteResults() throws { + let host = RecordingHost() + host.replies = [ + "{\"acknowledged\": false}", + "{\"acknowledged\": false}", + "{\"acknowledged\": false, \"insertedIds\": [{\"$numberInt\": \"7\"}]}", + "{\"acknowledged\": false, \"insertedIds\": [{\"$numberInt\": \"7\"}, {\"$numberInt\": \"8\"}]}", + "{\"acknowledged\": false}" + ] + let context = try makeContext(host) + let expected = [ + ("db.orders.updateMany({}, {$set: {b: 1}}, {writeConcern: {w: 0}})", "{\"acknowledged\":false}"), + ("db.orders.deleteOne({a: 1}, {writeConcern: {w: 0}})", "{\"acknowledged\":false}"), + ( + "db.orders.insertOne({_id: 7}, {writeConcern: {w: 0}})", + "{\"acknowledged\":false,\"insertedId\":{\"$numberInt\":\"7\"}}" + ), + ( + "db.orders.insertMany([{_id: 7}, {_id: 8}], {writeConcern: {w: 0}})", + "{\"acknowledged\":false,\"insertedIds\":[{\"$numberInt\":\"7\"},{\"$numberInt\":\"8\"}]}" + ), + ("db.orders.bulkWrite([{deleteOne: {filter: {}}}], {writeConcern: {w: 0}})", "{\"acknowledged\":false}") + ] + for (statement, result) in expected { + let value = context.evaluateScript("EJSON.stringify(\(statement))") + #expect(context.exception == nil, "\(statement) threw") + #expect(value?.toString() == result, "\(statement)") + } + } + + @Test("An acknowledged write still reports acknowledged true with its counts") + func acknowledgedWriteResults() throws { + let host = RecordingHost() + host.replies = ["{\"n\": 1}", "{\"insertedIds\": [{\"$numberInt\": \"7\"}], \"insertedCount\": 1}"] + let context = try makeContext(host) + + let deleted = context.evaluateScript("EJSON.stringify(db.orders.deleteOne({a: 1}))") + let inserted = context.evaluateScript("EJSON.stringify(db.orders.insertMany([{_id: 7}]))") + #expect(context.exception == nil) + #expect(deleted?.toString() == "{\"acknowledged\":true,\"deletedCount\":{\"$numberInt\":\"1\"}}") + #expect(inserted?.toString() + == "{\"acknowledged\":true,\"insertedIds\":[{\"$numberInt\":\"7\"}],\"insertedCount\":{\"$numberInt\":\"1\"}}") + } + + @Test("A write's failure reaches the script marked as a write's, with its stage") + func writeFailureCarriesItsStage() throws { + let host = RecordingHost() + let stopped = MongoWriteFailure(code: 50, message: "operation exceeded time limit", stage: .command) + host.refusals = ["update": MongoScriptJson.failure(stopped)] + let context = try makeContext(host) + + context.evaluateScript("db.orders.updateMany({}, {$set: {b: 1}})") + let exception = try #require(context.exception) + #expect(MongoScriptContext.writeFailure(in: exception) == stopped) + } + + @Test("A read that fails after the script caught a write's failure is not taken for the write") + func caughtWriteFailureDoesNotMarkALaterRead() throws { + let host = RecordingHost() + let stopped = MongoWriteFailure(code: 50, message: "operation exceeded time limit", stage: .command) + host.refusals = [ + "update": MongoScriptJson.failure(stopped), + "command": MongoScriptJson.failure(message: "operation exceeded time limit", code: 50) + ] + let context = try makeContext(host) + + context.evaluateScript(""" + try { db.orders.updateMany({}, {$set: {b: 1}}) } catch (e) {} + db.runCommand({count: "orders"}) + """) + let read = try #require(context.exception) + #expect(read.objectForKeyedSubscript("code")?.toInt32() == 50) + #expect(read.objectForKeyedSubscript("message")?.toString() == "operation exceeded time limit") + #expect(MongoScriptContext.writeFailure(in: read) == nil) + + context.exception = nil + context.evaluateScript("try { db.orders.updateMany({}, {$set: {b: 1}}) } catch (e) { throw e }") + let rethrown = try #require(context.exception) + #expect(MongoScriptContext.writeFailure(in: rethrown) == stopped) + } + + @Test("An error a script throws itself is never taken for a write's failure") + func scriptErrorIsNotAWrite() throws { + let context = try makeContext(RecordingHost()) + context.evaluateScript("var e = new Error('x'); e.code = 50; throw e") + let exception = try #require(context.exception) + #expect(MongoScriptContext.writeFailure(in: exception) == nil) + } + @Test("An aggregation pipeline crosses as an array of stages") func aggregatePipeline() throws { let host = RecordingHost() diff --git a/TableProTests/Plugins/MongoWriteFailureTests.swift b/TableProTests/Plugins/MongoWriteFailureTests.swift index 24b87c9533..0e06c4812d 100644 --- a/TableProTests/Plugins/MongoWriteFailureTests.swift +++ b/TableProTests/Plugins/MongoWriteFailureTests.swift @@ -14,7 +14,9 @@ struct MongoWriteFailureTests { "errmsg":"Document failed validation","errInfo":{"failingDocumentId":{"$oid":"6ab6cea9cc5310bead7fd537"},\ "details":{"operatorName":"$jsonSchema"}}}],"nModified":{"$numberInt":"0"},"ok":{"$numberInt":"1"}} """ - #expect(MongoWriteFailure.read(fromReply: reply) == MongoWriteFailure(code: 121, message: "Document failed validation")) + #expect(MongoWriteFailure.read(fromReply: reply) == MongoWriteFailure( + code: 121, message: "Document failed validation", stage: .document + )) } @Test("An update that would change _id is a failure") @@ -48,10 +50,66 @@ struct MongoWriteFailureTests { """ #expect(MongoWriteFailure.read(fromReply: reply) == MongoWriteFailure( code: 64, - message: MongoScriptText.writeNotAcknowledged(reason: "waiting for replication timed out") + message: MongoScriptText.writeNotAcknowledged(reason: "waiting for replication timed out"), + stage: .unconfirmed + )) + } + + @Test("The insert call's plural writeConcernErrors also says the write was applied") + func pluralWriteConcernErrors() { + let reply = """ + {"insertedCount":{"$numberInt":"1"},"writeConcernErrors":[{"code":{"$numberInt":"64"},\ + "errmsg":"waiting for replication timed out"}]} + """ + #expect(MongoWriteFailure.read(fromReply: reply) == MongoWriteFailure( + code: 64, + message: MongoScriptText.writeNotAcknowledged(reason: "waiting for replication timed out"), + stage: .unconfirmed )) } + @Test("A command the server stopped carries the server's own code and message") + func commandStoppedByTimeout() { + let reply = """ + { "ok" : { "$numberDouble" : "0.0" }, "errmsg" : "operation exceeded time limit", \ + "code" : { "$numberInt" : "50" }, "codeName" : "MaxTimeMSExpired" } + """ + let failure = MongoWriteFailure.read(fromReply: reply) + #expect(failure == MongoWriteFailure(code: 50, message: "operation exceeded time limit", stage: .command)) + #expect(failure?.stoppedWhileRunning == true) + } + + @Test("A command the server refused outright did not stop while running") + func commandRefused() { + let reply = """ + { "ok" : { "$numberDouble" : "0.0" }, "errmsg" : "cannot use 'w' > 1 when a host is not replicated", \ + "code" : { "$numberInt" : "2" }, "codeName" : "BadValue" } + """ + let failure = MongoWriteFailure.read(fromReply: reply) + #expect(failure?.stage == .command) + #expect(failure?.code == 2) + #expect(failure?.message == "cannot use 'w' > 1 when a host is not replicated") + #expect(failure?.stoppedWhileRunning == false) + } + + @Test("The insert call's errorReplies carry the server's refusal") + func errorReplies() { + let reply = """ + { "insertedCount" : { "$numberInt" : "0" }, "errorReplies" : [ { "ok" : { "$numberDouble" : "0.0" }, \ + "errmsg" : "cannot use 'w' > 1 when a host is not replicated", "code" : { "$numberInt" : "2" }, \ + "codeName" : "BadValue" } ] } + """ + #expect(MongoWriteFailure.read(fromReply: reply) == MongoWriteFailure( + code: 2, message: "cannot use 'w' > 1 when a host is not replicated", stage: .command + )) + } + + @Test("A reply with no server answer is left to the caller's error domain") + func noServerAnswer() { + #expect(MongoWriteFailure.read(fromReply: "{}") == nil) + #expect(MongoWriteFailure.read(fromReply: "{\"errorLabels\":[\"RetryableWriteError\"]}") == nil) + } + @Test("The first write error wins over a write concern error") func writeErrorsBeforeConcern() { let reply = """ @@ -87,17 +145,59 @@ struct MongoWriteFailureTests { } struct MongoScriptStatementFailureTests { - @Test("A failure with no database switch is passed on unchanged") + @Test("A failure with no database switch and no writes is passed on unchanged") func noSwitch() { let error = MongoScriptError("boom") - #expect(MongoScriptStatementFailure.carrying(error, databaseSwitch: nil) as? MongoScriptError == error) + let carried = MongoScriptStatementFailure.carrying(error, databaseSwitch: nil, writes: MongoWriteLedger()) + #expect(carried as? MongoScriptError == error) } @Test("A failure after a database switch carries the switch and keeps its message") func carriesSwitch() throws { - let carried = MongoScriptStatementFailure.carrying(MongoScriptError("boom"), databaseSwitch: "reports") + let carried = MongoScriptStatementFailure.carrying( + MongoScriptError("boom"), databaseSwitch: "reports", writes: MongoWriteLedger() + ) let failure = try #require(carried as? MongoScriptStatementFailure) #expect(failure.databaseSwitch == "reports") #expect(failure.localizedDescription == "boom") } + + @Test("A failure after a write carries the ledger and names what was written") + func carriesWrites() throws { + var writes = MongoWriteLedger() + writes.record( + .insert, touchesMany: false, acknowledged: true, + outcome: MongoWriteOutcome(replyJson: "{\"n\": 1}", failure: nil) + ) + let carried = MongoScriptStatementFailure.carrying(MongoScriptError("boom"), databaseSwitch: nil, writes: writes) + let failure = try #require(carried as? MongoScriptStatementFailure) + #expect(failure.databaseSwitch == nil) + #expect(failure.writes == writes) + #expect(failure.localizedDescription == "boom\n\n\(MongoScriptText.writesChanged(1))") + } + + @Test("A write that failed and changed nothing is passed on as itself, its stage with it") + func failedWriteWithNoChangeKeepsItsStage() throws { + var writes = MongoWriteLedger() + let stopped = MongoWriteFailure(code: 50, message: "operation exceeded time limit", stage: .command) + writes.record( + .update, touchesMany: false, acknowledged: true, + outcome: MongoWriteOutcome(replyJson: "{}", failure: stopped) + ) + let carried = MongoScriptStatementFailure.carrying(stopped, databaseSwitch: nil, writes: writes) + #expect(carried as? MongoWriteFailure == stopped) + #expect(carried.localizedDescription == "operation exceeded time limit") + } + + @Test("A failed write crosses the bridge with its stage") + func bridgeCarriesStage() throws { + let failure = MongoWriteFailure(code: 50, message: "operation \"exceeded\" time limit", stage: .unconfirmed) + let data = try #require(MongoScriptJson.failure(failure).data(using: .utf8)) + let response = try #require(try JSONSerialization.jsonObject(with: data) as? [String: Any]) + let error = try #require(response["e"] as? [String: Any]) + #expect(response["ok"] as? Bool == false) + #expect(error["m"] as? String == failure.message) + #expect(error["c"] as? Int == 50) + #expect(error["s"] as? String == "unconfirmed") + } } diff --git a/TableProTests/Plugins/MongoWriteLedgerTests.swift b/TableProTests/Plugins/MongoWriteLedgerTests.swift new file mode 100644 index 0000000000..a8084c98fa --- /dev/null +++ b/TableProTests/Plugins/MongoWriteLedgerTests.swift @@ -0,0 +1,354 @@ +// +// MongoWriteLedgerTests.swift +// TableProTests +// + +import Foundation +import Testing + +/// The replies below are the canonical Extended JSON libmongoc 1.28.1 handed back from MongoDB +/// 7.0.43, so the ledger is tested against shapes the host actually sees. +struct MongoWriteLedgerTests { + private static let validatorRejection = """ + { "n" : { "$numberInt" : "0" }, "writeErrors" : [ { "index" : { "$numberInt" : "0" }, \ + "code" : { "$numberInt" : "121" }, "errmsg" : "Document failed validation" } ], \ + "nModified" : { "$numberInt" : "0" }, "ok" : { "$numberDouble" : "1.0" } } + """ + + private static let rawInsertDuplicateAtTwo = """ + { "n" : { "$numberInt" : "2" }, "writeErrors" : [ { "index" : { "$numberInt" : "2" }, \ + "code" : { "$numberInt" : "11000" }, "errmsg" : "E11000 duplicate key error" } ], \ + "ok" : { "$numberDouble" : "1.0" } } + """ + + private static let crudInsertDuplicateAtTwo = """ + { "insertedCount" : { "$numberInt" : "2" }, "writeErrors" : [ { "index" : { "$numberInt" : "2" }, \ + "code" : { "$numberInt" : "11000" }, "errmsg" : "E11000 duplicate key error" } ] } + """ + + private static let maxTimeExpired = """ + { "ok" : { "$numberDouble" : "0.0" }, "errmsg" : "operation exceeded time limit", \ + "code" : { "$numberInt" : "50" }, "codeName" : "MaxTimeMSExpired" } + """ + + private static let interrupted = """ + { "ok" : { "$numberDouble" : "0.0" }, "errmsg" : "operation was interrupted", \ + "code" : { "$numberInt" : "11601" }, "codeName" : "Interrupted" } + """ + + private static let replicationRefused = """ + { "ok" : { "$numberDouble" : "0.0" }, "errmsg" : "cannot use 'w' > 1 when a host is not replicated", \ + "code" : { "$numberInt" : "2" }, "codeName" : "BadValue" } + """ + + private static func outcome(_ reply: String) -> MongoWriteOutcome { + MongoWriteOutcome(replyJson: reply, failure: MongoWriteFailure.read(fromReply: reply)) + } + + private static func ledger(_ operation: MongoWriteOperation, touchesMany: Bool, reply: String) -> MongoWriteLedger { + var ledger = MongoWriteLedger() + ledger.record(operation, touchesMany: touchesMany, acknowledged: true, outcome: outcome(reply)) + return ledger + } + + @Test("An updateMany that fails part-way may have changed documents its reply does not count") + func updateManyDocumentErrorMayHaveChanged() { + let ledger = Self.ledger(.update, touchesMany: true, reply: Self.validatorRejection) + #expect(ledger.changed == 0) + #expect(ledger.mayHaveChangedMore) + #expect(ledger.note == MongoScriptText.writesMayHaveChanged) + } + + @Test("An updateOne that fails changed nothing, so there is no note") + func updateOneDocumentErrorChangedNothing() { + let ledger = Self.ledger(.update, touchesMany: false, reply: Self.validatorRejection) + #expect(ledger.changed == 0) + #expect(!ledger.mayHaveChangedMore) + #expect(ledger.note == nil) + } + + @Test("An ordered insert counts the documents before the duplicate, from either reply shape") + func insertCountsWhatLanded() { + for reply in [Self.rawInsertDuplicateAtTwo, Self.crudInsertDuplicateAtTwo] { + let ledger = Self.ledger(.insert, touchesMany: true, reply: reply) + #expect(ledger.changed == 2) + #expect(!ledger.mayHaveChangedMore) + #expect(ledger.note == MongoScriptText.writesChanged(2)) + } + } + + @Test("A multi-document write stopped by a timeout or a kill may have changed documents") + func interruptedMultiWriteMayHaveChanged() { + for operation in [MongoWriteOperation.update, .delete] { + for reply in [Self.maxTimeExpired, Self.interrupted] { + let ledger = Self.ledger(operation, touchesMany: true, reply: reply) + #expect(ledger.mayHaveChangedMore) + #expect(MongoWriteFailure.read(fromReply: reply)?.stoppedWhileRunning == true) + #expect(ledger.note == MongoScriptText.writesMayHaveChanged) + } + } + } + + @Test("A single-document write stopped by a timeout changed nothing and leaves nothing to report") + func interruptedSingleWriteChangedNothing() { + let ledger = Self.ledger(.update, touchesMany: false, reply: Self.maxTimeExpired) + #expect(!ledger.mayHaveChangedMore) + #expect(ledger.note == nil) + #expect(ledger.isEmpty) + } + + @Test("A command the server refused outright changed nothing") + func refusedCommandChangedNothing() { + let ledger = Self.ledger(.update, touchesMany: true, reply: Self.replicationRefused) + #expect(ledger.changed == 0) + #expect(!ledger.mayHaveChangedMore) + #expect(ledger.note == nil) + } + + @Test("A bulkWrite counts every operation that finished before the one that failed") + func bulkSequenceMatchesMongosh() { + var ledger = MongoWriteLedger() + ledger.record(.insert, touchesMany: false, acknowledged: true, outcome: Self.outcome(""" + { "n" : { "$numberInt" : "1" }, "ok" : { "$numberDouble" : "1.0" } } + """)) + ledger.record(.update, touchesMany: true, acknowledged: true, outcome: Self.outcome(""" + { "n" : { "$numberInt" : "4" }, "nModified" : { "$numberInt" : "4" }, "ok" : { "$numberDouble" : "1.0" } } + """)) + ledger.record(.insert, touchesMany: false, acknowledged: true, outcome: Self.outcome(""" + { "n" : { "$numberInt" : "0" }, "writeErrors" : [ { "index" : { "$numberInt" : "0" }, \ + "code" : { "$numberInt" : "11000" }, "errmsg" : "E11000 duplicate key error" } ], \ + "ok" : { "$numberDouble" : "1.0" } } + """)) + #expect(ledger.changed == 5) + #expect(ledger.note == MongoScriptText.writesChanged(5)) + } + + @Test("A known count followed by an updateMany that failed part-way says both") + func knownCountAndPossiblyMore() { + var ledger = MongoWriteLedger() + ledger.record(.delete, touchesMany: true, acknowledged: true, outcome: Self.outcome(""" + { "n" : { "$numberInt" : "3" }, "ok" : { "$numberDouble" : "1.0" } } + """)) + ledger.record(.update, touchesMany: true, acknowledged: true, outcome: Self.outcome(Self.validatorRejection)) + #expect(ledger.note == MongoScriptText.writesChangedAndMaybeMore(3)) + } + + @Test("A write the servers did not confirm was still applied and counts") + func unconfirmedWriteCounts() { + let ledger = Self.ledger(.update, touchesMany: false, reply: """ + {"n":{"$numberInt":"1"},"nModified":{"$numberInt":"1"},"writeConcernError":{"code":{"$numberInt":"64"},\ + "errmsg":"waiting for replication timed out"},"ok":{"$numberDouble":"1.0"}} + """) + #expect(ledger.changed == 1) + #expect(ledger.note == MongoScriptText.writesChanged(1)) + } + + @Test("Upserts count as changes and a findAndModify counts the document it reached") + func upsertAndFindAndModifyCounts() { + var ledger = MongoWriteLedger() + ledger.record(.update, touchesMany: false, acknowledged: true, outcome: Self.outcome(""" + { "n" : { "$numberInt" : "1" }, "upserted" : [ { "index" : { "$numberInt" : "0" }, \ + "_id" : { "$numberInt" : "100" } } ], "nModified" : { "$numberInt" : "0" }, "ok" : { "$numberDouble" : "1.0" } } + """)) + ledger.record(.findAndModify, touchesMany: false, acknowledged: true, outcome: Self.outcome(""" + { "lastErrorObject" : { "n" : { "$numberInt" : "1" }, "updatedExisting" : true }, \ + "value" : { "_id" : { "$numberInt" : "1" } }, "ok" : { "$numberDouble" : "1.0" } } + """)) + #expect(ledger.changed == 2) + } + + @Test("A write whose connection broke may have been applied, and one never sent was not") + func unansweredAndNotSent() { + var unanswered = MongoWriteLedger() + unanswered.record(.update, touchesMany: false, acknowledged: true, outcome: MongoWriteOutcome( + replyJson: "{}", failure: MongoWriteFailure(code: 6, message: "socket error", stage: .unanswered) + )) + #expect(unanswered.note == MongoScriptText.writesMayHaveChanged) + + var notSent = MongoWriteLedger() + notSent.record(.update, touchesMany: true, acknowledged: true, outcome: MongoWriteOutcome( + replyJson: "{}", failure: MongoWriteFailure(code: 13_053, message: "No suitable servers", stage: .notSent) + )) + #expect(notSent.changed == 0) + #expect(notSent.note == nil) + } + + @Test("An insert that stopped before sending every document still counts the batches that went") + func notSentKeepsEarlierBatches() { + var ledger = MongoWriteLedger() + ledger.record(.insert, touchesMany: true, acknowledged: true, outcome: MongoWriteOutcome( + replyJson: "{ \"insertedCount\" : { \"$numberInt\" : \"3\" } }", + failure: MongoWriteFailure( + code: 0, message: MongoScriptText.insertStoppedAtOversizedDocument, stage: .stoppedBetweenBatches + ) + )) + #expect(ledger.changed == 3) + #expect(!ledger.mayHaveChangedMore) + #expect(ledger.note == MongoScriptText.writesChanged(3)) + } + + @Test("An unacknowledged insert that stopped between batches may have written the batches that went") + func unacknowledgedStopBetweenBatchesMayHaveChanged() { + var ledger = MongoWriteLedger() + ledger.record(.insert, touchesMany: true, acknowledged: false, outcome: MongoWriteOutcome( + replyJson: "{ }", + failure: MongoWriteFailure( + code: 0, message: MongoScriptText.insertStoppedAtOversizedDocument, stage: .stoppedBetweenBatches + ) + )) + #expect(ledger.changed == 0) + #expect(ledger.mayHaveChangedMore) + #expect(ledger.note == MongoScriptText.writesMayHaveChanged) + } + + @Test("A multi-document write stopped by an interruption code from MongoDB 8.0 on may have changed documents") + func newerInterruptionsMayHaveChanged() { + let codes: [UInt32] = [91_331, 10_045_600, 453, 471, 473, 485, 509] + for code in codes { + var ledger = MongoWriteLedger() + ledger.record(.update, touchesMany: true, acknowledged: true, outcome: Self.outcome(""" + { "ok" : { "$numberDouble" : "0.0" }, "errmsg" : "interrupted", "code" : { "$numberInt" : "\(code)" } } + """)) + #expect(ledger.mayHaveChangedMore, "\(code)") + #expect(ledger.note == MongoScriptText.writesMayHaveChanged, "\(code)") + } + } + + @Test("A write sent with w: 0 may have changed documents whatever its reply counts") + func unacknowledgedWriteMayHaveChanged() { + let commandReply = """ + { "n" : { "$numberInt" : "0" }, "nModified" : { "$numberInt" : "0" }, "ok" : { "$numberDouble" : "1.0" } } + """ + for (operation, reply) in [(MongoWriteOperation.update, commandReply), (.delete, commandReply), (.insert, "{ }")] { + var ledger = MongoWriteLedger() + ledger.record(operation, touchesMany: false, acknowledged: false, outcome: Self.outcome(reply)) + #expect(ledger.changed == 0) + #expect(ledger.mayHaveChangedMore) + #expect(ledger.note == MongoScriptText.writesMayHaveChanged) + } + } + + @Test("An unacknowledged write refused on this side changed nothing") + func unacknowledgedWriteNotSent() { + var ledger = MongoWriteLedger() + ledger.record(.insert, touchesMany: true, acknowledged: false, outcome: MongoWriteOutcome( + replyJson: "{ }", failure: MongoWriteFailure(code: 22, message: "Invalid writeConcern", stage: .notSent) + )) + #expect(ledger.note == nil) + #expect(ledger.isEmpty) + } + + @Test("A write still waiting on the server when the statement is abandoned may have been applied") + func writeInFlight() { + var ledger = MongoWriteLedger() + ledger.beginWrite() + #expect(ledger.note == MongoScriptText.writesMayHaveChanged) + ledger.abandonWrite() + #expect(ledger.note == nil) + #expect(ledger.isEmpty) + } +} + +struct MongoWriteLedgerMessageTests { + private static let timeoutText = "operation exceeded time limit" + + private static func ledger(recording failure: MongoWriteFailure, touchesMany: Bool) -> MongoWriteLedger { + var ledger = MongoWriteLedger() + ledger.record( + .update, touchesMany: touchesMany, acknowledged: true, + outcome: MongoWriteOutcome(replyJson: "{}", failure: failure) + ) + return ledger + } + + @Test("A write stopped by the timeout reads as a write, with the note after the timeout text") + func writeTimeoutKeepsNote() { + let ledger = Self.ledger( + recording: MongoWriteFailure(code: 50, message: Self.timeoutText, stage: .command), touchesMany: true + ) + #expect(ledger.reportedMessage( + code: 50, message: Self.timeoutText, failedWrite: .command, maxTimeMS: 30_000 + ) == """ + \(MongoDBTimeoutPolicy.writeTimeoutMessage(maxTimeMS: 30_000)) + + \(MongoScriptText.writesMayHaveChanged) + """) + } + + @Test("A read that times out after the script caught a write's timeout reads as a query, not a write") + func readTimeoutAfterCaughtWriteTimeout() { + let ledger = Self.ledger( + recording: MongoWriteFailure(code: 50, message: Self.timeoutText, stage: .command), touchesMany: true + ) + #expect(ledger.reportedMessage( + code: 50, message: Self.timeoutText, failedWrite: nil, maxTimeMS: 1_000 + ) == """ + \(MongoDBTimeoutPolicy.timeoutMessage(maxTimeMS: 1_000)) + + \(MongoScriptText.writesMayHaveChanged) + """) + } + + @Test("A single write stopped by the timeout reads as a write with nothing recorded before it") + func singleWriteTimeout() { + #expect(MongoWriteLedger().reportedMessage( + code: 50, message: Self.timeoutText, failedWrite: .command, maxTimeMS: 1_000 + ) == MongoDBTimeoutPolicy.writeTimeoutMessage(maxTimeMS: 1_000)) + } + + @Test("A read timeout with nothing written keeps the read wording and no note") + func readTimeout() { + #expect(MongoWriteLedger().reportedMessage( + code: 50, message: Self.timeoutText, failedWrite: nil, maxTimeMS: 30_000 + ) == MongoDBTimeoutPolicy.timeoutMessage(maxTimeMS: 30_000)) + } + + @Test("A read that times out after an earlier write keeps the read wording and adds the note") + func readTimeoutAfterWrite() { + var ledger = MongoWriteLedger() + ledger.record(.delete, touchesMany: true, acknowledged: true, outcome: MongoWriteOutcome(replyJson: "{\"n\": 2}", failure: nil)) + #expect(ledger.reportedMessage(code: 50, message: Self.timeoutText, failedWrite: nil, maxTimeMS: 1_000) == """ + \(MongoDBTimeoutPolicy.timeoutMessage(maxTimeMS: 1_000)) + + \(MongoScriptText.writesChanged(2)) + """) + } + + @Test("A write-concern error with the timeout's code keeps the text that says the write was applied") + func unconfirmedTimeoutKeepsItsText() { + let applied = MongoScriptText.writeNotAcknowledged(reason: "waiting for replication timed out") + var ledger = MongoWriteLedger() + ledger.record(.update, touchesMany: false, acknowledged: true, outcome: MongoWriteOutcome( + replyJson: "{\"n\": 1, \"nModified\": 1}", + failure: MongoWriteFailure(code: 50, message: applied, stage: .unconfirmed) + )) + #expect(ledger.reportedMessage(code: 50, message: applied, failedWrite: .unconfirmed, maxTimeMS: 30_000) == """ + \(applied) + + \(MongoScriptText.writesChanged(1)) + """) + } + + @Test("A server refusal keeps its message and gets the count of what was already written") + func documentErrorWithCount() { + var ledger = MongoWriteLedger() + ledger.record(.insert, touchesMany: true, acknowledged: true, outcome: MongoWriteOutcome( + replyJson: "{\"n\": 2}", + failure: MongoWriteFailure(code: 121, message: "Document failed validation", stage: .document) + )) + #expect(ledger.reportedMessage( + code: 121, message: "Document failed validation", failedWrite: .document, maxTimeMS: 30_000 + ) == """ + Document failed validation + + \(MongoScriptText.writesChanged(2)) + """) + } + + @Test("With no query timeout set, a timeout code is left as the server wrote it") + func noTimeoutConfigured() { + #expect(MongoWriteLedger().reportedMessage( + code: 50, message: Self.timeoutText, failedWrite: .command, maxTimeMS: nil + ) == Self.timeoutText) + } +} diff --git a/docs/databases/mongodb.mdx b/docs/databases/mongodb.mdx index 8145037bd8..e66c43c80e 100644 --- a/docs/databases/mongodb.mdx +++ b/docs/databases/mongodb.mdx @@ -37,6 +37,8 @@ Naming a **Database** skips listing every database on the server, which is worth Also in Advanced: **Read Preference**, **Write Concern**, **Use SRV Record**, **Replica Set** name, and **Legacy UUID Encoding**. There is no minimum server version; the driver adapts what it asks for to what the server answers. +**Write Concern** covers every insert, update and delete, from a query tab or from a save in the data grid, unless the statement passes its own `writeConcern`. Index, collection and database commands, and `db.runCommand`, go to the server as written. On a server that is not a replica set, pick **Default**, **Majority** or **1**: with **2** or **3** every write fails with `cannot use 'w' > 1 when a host is not replicated`. `journal` and `wtimeoutMS` in a pasted connection URL go out with every write too. + MongoDB connection form with the multi-host Hosts editor MongoDB connection form with the multi-host Hosts editor @@ -176,14 +178,33 @@ or set the modifier before the first read. ### Write options `updateOne`, `updateMany`, `replaceOne`, `findOneAndUpdate` and the delete calls take an options -document, and `upsert`, `arrayFilters`, `hint`, `collation` and `returnDocument` reach the server. A -write returns the object mongosh returns: `matchedCount`, `modifiedCount`, `upsertedCount` and +document, and `upsert`, `arrayFilters`, `hint`, `collation`, `returnDocument` and `writeConcern` +reach the server. `insertOne`, `insertMany`, `insert` and `bulkWrite` take `writeConcern` too, and +`insertMany` takes `ordered: false` to go on inserting past a document that fails. +`remove(filter, true)` and `remove(filter, {justOne: true})` delete one matching document. A write +returns the object mongosh returns: `matchedCount`, `modifiedCount`, `upsertedCount` and `upsertedId` for an update, `deletedCount` for a delete, `insertedId` for an insert. ```javascript db.users.updateOne({_id: 1}, {$set: {active: true}}, {upsert: true}) +db.events.insertMany(batch, {ordered: false, writeConcern: {w: "majority"}}) ``` +`writeConcern` takes mongosh's `journal` and `wtimeoutMS` as well as `j` and `wtimeout`. Naming any +of `w`, `j` or `wtimeout` replaces the connection's **Write Concern** for that write; an empty +`writeConcern: {}` keeps it. With `w: 0` and no `j: true` the server does not report what a write +did, so the result is `{acknowledged: false}` with no counts. `findOneAndUpdate` and the other +find-and-modify calls still return the document. + +### When a write stops part-way + +MongoDB keeps whatever a write changed before it failed, timed out or was killed, and undoes +nothing. An `updateMany` that fails on its third document has already changed the first two. The +error names this under the server's message: `insertMany` and `bulkWrite` give the count, and +`updateMany` and `deleteMany` say only that some documents may have changed, as does any write sent +with `w: 0`. A statement that ran several writes counts them all, and so does an export whose query +fails after the writes before it. Check the data before you run the statement again. + ### Methods Collection: `find`, `findOne`, `aggregate`, `countDocuments`/`count`, `estimatedDocumentCount`, @@ -219,6 +240,7 @@ New connections default to **Disabled**, and the driver has no TLS fallback: **P - GridFS buckets are not browsable, and change streams are unsupported. - A script that loops without touching the database cannot be stopped: JavaScriptCore has no public way to interrupt one. `Cmd+.` stops anything that reads, writes or prints, which covers every query. A script silent for 120 seconds is abandoned and the shell restarts. - Field names that look like integers (`"0"`, `"12"`) sort ahead of the rest in a document literal, which is what JavaScript does with them. +- `bulkWrite` runs its operations in order and stops at the first one that fails, even with `ordered: false`. To go on past a failed insert, use `insertMany` with `ordered: false`. ## Troubleshooting diff --git a/project.yml b/project.yml index ecd82d75d7..7a1b898fe4 100644 --- a/project.yml +++ b/project.yml @@ -532,6 +532,7 @@ targets: - Plugins/MongoDBDriverPlugin/MongoScriptPrelude.swift - Plugins/MongoDBDriverPlugin/MongoScriptText.swift - Plugins/MongoDBDriverPlugin/MongoWriteFailure.swift + - Plugins/MongoDBDriverPlugin/MongoWriteLedger.swift - Plugins/MongoDBDriverPlugin/MongoStreamProjection.swift - Plugins/OracleDriverPlugin/OracleObjectQueries.swift - Plugins/MySQLDriverPlugin/GeometryWKBParser.swift diff --git a/scripts/check-mongodb-interruption-codes.sh b/scripts/check-mongodb-interruption-codes.sh new file mode 100755 index 0000000000..950ee6c41a --- /dev/null +++ b/scripts/check-mongodb-interruption-codes.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# Compare the MongoDB plugin's interruption codes against the server's own error code tables. +# +# A multi-document write that fails with ok: 0 and a code in the server's Interruption category +# may already have changed documents: the server fails the whole batch for exactly those codes, +# after writing the documents before the one it stopped at, and its reply does not say how many. +# The plugin reports that from MongoDBServerErrorCode.interruptionCategory, a hand-copied union of +# the category across every release. Releases keep adding to it (8.0 to 8.3 added six codes), and +# a code missing from the set reads as a write that changed nothing, so this diffs the set against +# every release branch of mongodb/mongo from 4.0 on. +# +# Usage: +# scripts/check-mongodb-interruption-codes.sh +# +# Needs curl, git, python3 and network access to GitHub. Exits non-zero on a disagreement. + +set -uo pipefail + +SOURCE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift" +REPO="https://github.com/mongodb/mongo" +RAW="https://raw.githubusercontent.com/mongodb/mongo" + +for tool in curl git python3; do + command -v "$tool" > /dev/null || { + echo "$tool not found" >&2 + exit 3 + } +done +[ -f "$SOURCE" ] || { + echo "not found: $SOURCE" >&2 + exit 3 +} + +# A private directory, matching every sibling check script. /tmp is world-writable, so a fixed +# name is something another local user can pre-create and control. +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +git ls-remote --heads "$REPO" 'v[0-9]*' > "$WORK/heads" || { + echo "could not list the branches of $REPO" >&2 + exit 3 +} +BRANCHES=() +while IFS= read -r branch; do + BRANCHES+=("$branch") +done < <(sed -n 's#.*refs/heads/\(v[0-9][0-9]*\.[0-9][0-9]*\)$#\1#p' "$WORK/heads" \ + | awk -F'[v.]' '$2 >= 4' \ + | sort -t. -k1.2,1n -k2,2n) +[ "${#BRANCHES[@]}" -gt 0 ] || { + echo "no release branches from 4.0 on in $REPO" >&2 + exit 3 +} + +for branch in "${BRANCHES[@]}"; do + if curl -fsS -o "$WORK/$branch.yml" "$RAW/$branch/src/mongo/base/error_codes.yml" 2> /dev/null; then + continue + fi + rm -f "$WORK/$branch.yml" + curl -fsS -o "$WORK/$branch.err" "$RAW/$branch/src/mongo/base/error_codes.err" || { + echo "no error code table on $branch" >&2 + exit 3 + } +done + +echo "Checking the interruption codes against ${#BRANCHES[@]} release branches" + +python3 - "$SOURCE" "$WORK" "${BRANCHES[@]}" << 'PY' +import os +import re +import sys + +source, work, branches = sys.argv[1], sys.argv[2], sys.argv[3:] + + +def uncommented(path): + return re.sub(r"#.*", "", open(path).read()) + + +def interruption_codes(branch): + yml = os.path.join(work, branch + ".yml") + if os.path.exists(yml): + text = uncommented(yml) + codes = {} + for body in re.findall(r"-\s*\{(.*?)\}", text, re.S): + code = re.search(r"\bcode:\s*(\d+)", body) + name = re.search(r"\bname:\s*(\w+)", body) + categories = re.search(r"\bcategories:\s*\[(.*?)\]", body, re.S) + if code and name and categories and "Interruption" in re.findall(r"\w+", categories.group(1)): + codes[int(code.group(1))] = name.group(1) + return codes + text = uncommented(os.path.join(work, branch + ".err")) + numbers = {name: int(code) for name, code in re.findall(r'error_code\("(\w+)",\s*(\d+)', text)} + listed = re.search(r'error_class\("Interruption",\s*\[(.*?)\]\)', text, re.S) + return {numbers[name]: name for name in re.findall(r'"(\w+)"', listed.group(1))} + + +swift = open(source).read() +constants = {name: int(value.replace("_", "")) for name, value in re.findall(r"static let (\w+): UInt32 = ([\d_]+)", swift)} +listed = re.search(r"interruptionCategory: Set = \[(.*?)\]", swift, re.S) +if listed is None: + print("interruptionCategory not found in " + source, file=sys.stderr) + sys.exit(3) +ours = set() +for token in re.findall(r"[\w]+", listed.group(1)): + ours.add(int(token.replace("_", "")) if token[0].isdigit() else constants[token]) + +first_seen = {} +for branch in branches: + for code, name in interruption_codes(branch).items(): + first_seen.setdefault(code, (name, branch)) + +missing = sorted(set(first_seen) - ours) +extra = sorted(ours - set(first_seen)) +for code in missing: + name, branch = first_seen[code] + print(f"missing: {code} {name} (Interruption since {branch})") +for code in extra: + print(f"not an Interruption code on any release branch: {code}") +if missing or extra: + sys.exit(1) +print(f"ok: {len(ours)} codes, the union across {branches[0]} to {branches[-1]}") +PY