diff --git a/CHANGELOG.md b/CHANGELOG.md index 56fdef005..1f28d12c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - PostgreSQL `money` values written as null in Parquet exports. - Files left behind when a multi-table Parquet export is stopped between tables. - Filter-bar BETWEEN refused on Typesense and Weaviate, and given the wrong lower bound on BigQuery. +- SurrealDB between, matches regex, is empty and raw filters run as equality, and is not empty showing only empty rows. +- SurrealDB edits to `in` and `out` dropped without a word when the same row had another edit. - Cassandra filter error telling MCP clients to use a Match All control they do not have. - Japanese, Chinese and Korean text in CSV, TSV and SQL files opening as garbled characters. - Every row of a UTF-16 data file rewritten on save. diff --git a/Packages/TableProCore/Tests/TableProSyncTests/SyncRecordMapperTests.swift b/Packages/TableProCore/Tests/TableProSyncTests/SyncRecordMapperTests.swift index 0463cdcad..2fa2afdfb 100644 --- a/Packages/TableProCore/Tests/TableProSyncTests/SyncRecordMapperTests.swift +++ b/Packages/TableProCore/Tests/TableProSyncTests/SyncRecordMapperTests.swift @@ -86,4 +86,25 @@ struct SyncRecordMapperTests { let decoded = try #require(SyncRecordMapper.toConnection(record)) #expect(decoded.safeModeLevel == .confirmWrites) } + + @Test("An unrecognized wire value keeps the legacy read-only restriction") + func unknownWireValuePreservesReadOnly() throws { + let record = makeRawRecord(safeModeLevelRaw: "someFutureLevel", isReadOnly: true) + let decoded = try #require(SyncRecordMapper.toConnection(record)) + #expect(decoded.safeModeLevel == .readOnly) + } + + @Test("A rename preserves an unrecognized wire value and requires confirmation") + func renamePreservesUnknownWireValue() throws { + let record = makeRawRecord(safeModeLevelRaw: "someFutureLevel") + var connection = try #require(SyncRecordMapper.toConnection(record)) + connection.name = "Renamed" + + SyncRecordMapper.updateRecord(record, with: connection) + + #expect(record["safeModeLevel"] as? String == "someFutureLevel") + let decoded = try #require(SyncRecordMapper.toConnection(record)) + #expect(decoded.name == "Renamed") + #expect(decoded.safeModeLevel == .confirmWrites) + } } diff --git a/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift b/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift index 613ba9a90..5517bec2f 100644 --- a/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift +++ b/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift @@ -264,23 +264,7 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { func buildFilteredQuery( table: String, schema: String?, - filters: [(column: String, op: String, value: String)], - logicMode: String, - sortColumns: [(columnIndex: Int, ascending: Bool)], - columns: [String], - limit: Int, - offset: Int - ) -> String? { - buildFilteredQuery( - table: table, schema: schema, filters: filters, logicMode: logicMode, - sortColumns: sortColumns, columns: columns, limit: limit, offset: offset, columnKinds: [:] - ) - } - - func buildFilteredQuery( - table: String, - schema: String?, - filters: [(column: String, op: String, value: String)], + queryFilters: [PluginQueryFilter], logicMode: String, sortColumns: [(columnIndex: Int, ascending: Bool)], columns: [String], @@ -291,7 +275,7 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { SurrealQueryBuilder.filtered( table: table, scope: scope(forSchema: schema), - filters: filters, + filters: queryFilters, logicMode: logicMode, sortColumns: Self.sorts(sortColumns, columns: columns), limit: limit, @@ -302,16 +286,25 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { func fetchFilteredRowCount( table: String, - filters: [(column: String, op: String, value: String)], + queryFilters: [PluginQueryFilter], + logicMode: String + ) async throws -> Int? { + try await count(table: table, schema: nil, filters: queryFilters, logicMode: logicMode) + } + + func fetchExactRowCount( + table: String, + schema: String?, + queryFilters: [PluginQueryFilter], logicMode: String ) async throws -> Int? { - try await count(table: table, schema: nil, filters: filters, logicMode: logicMode) + try await count(table: table, schema: schema, filters: queryFilters, logicMode: logicMode) } func count( table: String, schema: String?, - filters: [(column: String, op: String, value: String)], + filters: [PluginQueryFilter], logicMode: String ) async throws -> Int? { let scope = scope(forSchema: schema) diff --git a/Plugins/SurrealDBDriverPlugin/SurrealQueryBuilder.swift b/Plugins/SurrealDBDriverPlugin/SurrealQueryBuilder.swift index 6397b5cf8..eb661934d 100644 --- a/Plugins/SurrealDBDriverPlugin/SurrealQueryBuilder.swift +++ b/Plugins/SurrealDBDriverPlugin/SurrealQueryBuilder.swift @@ -28,7 +28,31 @@ public struct SurrealScope: Equatable, Sendable { } } +public struct SurrealFilterRefusal: Error, Equatable, Sendable { + public let message: String + + public static let incompleteRange = SurrealFilterRefusal( + message: String(localized: "Enter both bounds to filter with BETWEEN.") + ) + + public static let rawConditionNotReadOnly = SurrealFilterRefusal( + message: String( + localized: "A raw SurrealDB filter must be one condition that only reads. Run anything else in the SurrealQL editor." + ) + ) + + public static func unsupportedOperator(_ op: String) -> SurrealFilterRefusal { + SurrealFilterRefusal(message: String(format: String(localized: "SurrealDB cannot filter rows with %@."), op)) + } + + public var statement: String { + "THROW " + SurrealQL.stringLiteral(message) + ";" + } +} + public enum SurrealQueryBuilder { + public static let rawFilterColumn = "__RAW__" + public static func browse( table: String, scope: SurrealScope, @@ -42,32 +66,40 @@ public enum SurrealQueryBuilder { public static func filtered( table: String, scope: SurrealScope, - filters: [(column: String, op: String, value: String)], + filters: [PluginQueryFilter], logicMode: String, sortColumns: [(column: String, ascending: Bool)], limit: Int, offset: Int, columnKinds: [String: PluginColumnKind] = [:] ) -> String { - let clause = whereClause(filters: filters, logicMode: logicMode, columnKinds: columnKinds) - return compose( - scope: scope, - statement: select(table: table, where: clause, sortColumns: sortColumns, limit: limit, offset: offset) - ) + do throws(SurrealFilterRefusal) { + let clause = try whereClause(filters: filters, logicMode: logicMode, columnKinds: columnKinds) + return compose( + scope: scope, + statement: select(table: table, where: clause, sortColumns: sortColumns, limit: limit, offset: offset) + ) + } catch { + return compose(scope: scope, statement: error.statement) + } } public static func count( table: String, scope: SurrealScope, - filters: [(column: String, op: String, value: String)], + filters: [PluginQueryFilter], logicMode: String ) -> String { - var statement = "SELECT count() AS total FROM " + SurrealQL.quoteIdentifier(table) - if let clause = whereClause(filters: filters, logicMode: logicMode) { - statement += " WHERE " + clause + do throws(SurrealFilterRefusal) { + var statement = "SELECT count() AS total FROM " + SurrealQL.quoteIdentifier(table) + if let clause = try whereClause(filters: filters, logicMode: logicMode) { + statement += " WHERE " + clause + } + statement += " GROUP ALL;" + return compose(scope: scope, statement: statement) + } catch { + return compose(scope: scope, statement: error.statement) } - statement += " GROUP ALL;" - return compose(scope: scope, statement: statement) } public static func sample(table: String, scope: SurrealScope, limit: Int) -> String { @@ -112,20 +144,27 @@ public enum SurrealQueryBuilder { } public static func whereClause( - filters: [(column: String, op: String, value: String)], + filters: [PluginQueryFilter], logicMode: String, columnKinds: [String: PluginColumnKind] = [:] - ) -> String? { - let conditions = filters.compactMap { condition($0, kind: columnKinds[$0.column]) } + ) throws(SurrealFilterRefusal) -> String? { + var conditions: [String] = [] + for filter in filters { + guard let condition = try condition(filter, kind: columnKinds[filter.column]) else { continue } + conditions.append(condition) + } guard !conditions.isEmpty else { return nil } let separator = logicMode.lowercased() == "or" ? " OR " : " AND " return conditions.joined(separator: separator) } private static func condition( - _ filter: (column: String, op: String, value: String), + _ filter: PluginQueryFilter, kind: PluginColumnKind? - ) -> String? { + ) throws(SurrealFilterRefusal) -> String? { + if filter.column == rawFilterColumn { + return try SurrealRawCondition.parenthesized(filter.value) + } guard !filter.column.isEmpty else { return nil } let column = SurrealQL.quoteIdentifier(filter.column) let op = filter.op.uppercased().trimmingCharacters(in: .whitespaces) @@ -136,6 +175,10 @@ public enum SurrealQueryBuilder { return "(\(column) = NONE OR \(column) = NULL)" case "IS NOT NULL": return "(\(column) != NONE AND \(column) != NULL)" + case "IS EMPTY": + return "(\(column) = NONE OR \(column) = NULL OR \(column) = '')" + case "IS NOT EMPTY": + return "(\(column) != NONE AND \(column) != NULL AND \(column) != '')" case "CONTAINS": return "string::contains( \(column), \(SurrealQL.stringLiteral(value)))" case "NOT CONTAINS": @@ -144,17 +187,56 @@ public enum SurrealQueryBuilder { return "string::starts_with( \(column), \(SurrealQL.stringLiteral(value)))" case "ENDS WITH": return "string::ends_with( \(column), \(SurrealQL.stringLiteral(value)))" + case "REGEX": + let match = "string::matches( \(column), \(SurrealQL.stringLiteral(value)))" + return "(\(column) != NONE AND \(column) != NULL AND \(match))" case "IN": return "\(column) INSIDE \(listLiteral(value, kind: kind))" case "NOT IN": return "\(column) NOTINSIDE \(listLiteral(value, kind: kind))" + case "BETWEEN": + let bounds = try rangeBounds(filter) + let lower = literal(bounds.lower, kind: kind) + let upper = literal(bounds.upper, kind: kind) + return "(\(column) >= \(lower) AND \(column) <= \(upper))" case "=", "!=", ">", ">=", "<", "<=": return "\(column) \(op) \(literal(value, kind: kind))" case "LIKE": return "string::contains( \(column), \(SurrealQL.stringLiteral(unwrapWildcards(value))))" default: - return "\(column) = \(literal(value, kind: kind))" + throw SurrealFilterRefusal.unsupportedOperator(filter.op) + } + } + + private static func rangeBounds( + _ filter: PluginQueryFilter + ) throws(SurrealFilterRefusal) -> (lower: String, upper: String) { + if let upper = filter.secondValue { + return try completeRange(lower: lowerBound(of: filter.value, upperBound: upper), upper: upper) } + let scalars = filter.value.unicodeScalars + guard let separator = scalars.firstIndex(of: ",") else { throw .incompleteRange } + return try completeRange( + lower: String(scalars[.. String { + let joinedSuffix = ("," + upperBound).unicodeScalars + let scalars = joinedValue.unicodeScalars + guard scalars.reversed().starts(with: joinedSuffix.reversed()) else { return joinedValue } + return String(scalars.dropLast(joinedSuffix.count)) + } + + private static func completeRange( + lower: String, + upper: String + ) throws(SurrealFilterRefusal) -> (lower: String, upper: String) { + let lowerBound = lower.trimmingCharacters(in: .whitespaces) + let upperBound = upper.trimmingCharacters(in: .whitespaces) + guard !lowerBound.isEmpty, !upperBound.isEmpty else { throw .incompleteRange } + return (lowerBound, upperBound) } private static func listLiteral(_ value: String, kind: PluginColumnKind?) -> String { diff --git a/Plugins/SurrealDBDriverPlugin/SurrealRawCondition.swift b/Plugins/SurrealDBDriverPlugin/SurrealRawCondition.swift new file mode 100644 index 000000000..2ab06637c --- /dev/null +++ b/Plugins/SurrealDBDriverPlugin/SurrealRawCondition.swift @@ -0,0 +1,135 @@ +// +// SurrealRawCondition.swift +// SurrealDBDriverPlugin +// + +import Foundation + +internal enum SurrealRawCondition { + static func parenthesized(_ text: String) throws(SurrealFilterRefusal) -> String? { + let condition = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !condition.isEmpty else { return nil } + guard onlyReads(Array(condition.unicodeScalars)) else { throw .rawConditionNotReadOnly } + return "(" + condition + ")" + } + + private static let writingKeywords: Set = [ + "ACCESS", "ALTER", "CREATE", "DEFINE", "DELETE", "INSERT", "KILL", "LIVE", "OPTION", + "REBUILD", "RELATE", "REMOVE", "UPDATE", "UPSERT", "USE" + ] + + private static let writingFunctionNamespaces: Set = ["api", "file", "fn", "http", "sequence"] + + private static func onlyReads(_ scalars: [Unicode.Scalar]) -> Bool { + var index = 0 + var depth = 0 + while index < scalars.count { + let scalar = scalars[index] + if let closing = closingDelimiter(of: scalar) { + guard let end = endOfQuoted(scalars, from: index + 1, closing: closing) else { return false } + index = end + 1 + continue + } + if isWordScalar(scalar) { + let end = endOfWord(scalars, from: index) + guard wordOnlyReads(scalars, start: index, end: end) else { return false } + index = end + continue + } + guard !startsStatementOrComment(scalars, at: index) else { return false } + if scalar == "(" { + depth += 1 + } else if scalar == ")" { + depth -= 1 + guard depth >= 0 else { return false } + } + index += 1 + } + return depth == 0 + } + + private static func closingDelimiter(of scalar: Unicode.Scalar) -> Unicode.Scalar? { + switch scalar { + case "'", "\"", "`": + return scalar + case "\u{27E8}": + return "\u{27E9}" + default: + return nil + } + } + + private static func endOfQuoted(_ scalars: [Unicode.Scalar], from start: Int, closing: Unicode.Scalar) -> Int? { + var index = start + while index < scalars.count { + if scalars[index] == "\\" { + index += 2 + continue + } + if scalars[index] == closing { + return index + } + index += 1 + } + return nil + } + + private static func isWordScalar(_ scalar: Unicode.Scalar) -> Bool { + scalar == "_" || CharacterSet.alphanumerics.contains(scalar) + } + + private static func endOfWord(_ scalars: [Unicode.Scalar], from start: Int) -> Int { + var index = start + while index < scalars.count, isWordScalar(scalars[index]) { + index += 1 + } + return index + } + + private static func wordOnlyReads(_ scalars: [Unicode.Scalar], start: Int, end: Int) -> Bool { + let word = String(String.UnicodeScalarView(scalars[start.. Bool { + guard start > 0 else { return false } + switch scalars[start - 1] { + case "$": + return true + case ".": + return start >= 2 && scalars[start - 2] != "." + case ":": + return start >= 3 && scalars[start - 2] == ":" && isWordScalar(scalars[start - 3]) + default: + return false + } + } + + private static func isFollowedByPathSeparator(_ scalars: [Unicode.Scalar], at index: Int) -> Bool { + var next = index + while next < scalars.count, CharacterSet.whitespacesAndNewlines.contains(scalars[next]) { + next += 1 + } + return next + 1 < scalars.count && scalars[next] == ":" && scalars[next + 1] == ":" + } + + private static func startsStatementOrComment(_ scalars: [Unicode.Scalar], at index: Int) -> Bool { + let next = index + 1 < scalars.count ? scalars[index + 1] : nil + switch scalars[index] { + case ";", "#": + return true + case "-": + return next == "-" + case "/": + return next == "/" || next == "*" + default: + return false + } + } +} diff --git a/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift b/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift index 859b75328..676fe2cbf 100644 --- a/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift +++ b/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift @@ -24,7 +24,7 @@ public enum SurrealStatementGenerator { insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set - ) throws -> [PluginRowWrite] { + ) throws(PluginRowWriteRefusal) -> [PluginRowWrite] { var writes: [PluginRowWrite] = [] for change in changes where change.type == .update && !insertedRowIndices.contains(change.rowIndex) { @@ -58,11 +58,12 @@ public enum SurrealStatementGenerator { columns: [String], kinds: [String: SurrealFieldKind], change: PluginRowChange - ) throws -> PluginRowWrite? { - guard let record = recordId(table: table, columns: columns, originalRow: change.originalRow) else { return nil } - let editable = change.cellChanges.filter { - !SurrealInfoParser.isReservedColumn($0.columnName) && !Self.isAutoDefault($0.newValue) + ) throws(PluginRowWriteRefusal) -> PluginRowWrite? { + if let reserved = change.cellChanges.first(where: { SurrealInfoParser.isReservedColumn($0.columnName) }) { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: reservedColumnReason(reserved.columnName)) } + guard let record = recordId(table: table, columns: columns, originalRow: change.originalRow) else { return nil } + let editable = change.cellChanges.filter { !Self.isAutoDefault($0.newValue) } guard !editable.isEmpty else { return nil } var parameters: [PluginCellValue] = [SurrealCellCoder.parameter(.recordId(record))] @@ -93,7 +94,7 @@ public enum SurrealStatementGenerator { kinds: [String: SurrealFieldKind], values: [PluginCellValue], rowIndex: Int - ) throws -> PluginRowWrite { + ) throws(PluginRowWriteRefusal) -> PluginRowWrite { var parameters: [PluginCellValue] = [] var assignments: [String] = [] var target = SurrealQL.quoteIdentifier(table) @@ -146,7 +147,7 @@ public enum SurrealStatementGenerator { // MARK: - Refusals - private static func refuseShortened(_ cell: PluginCellValue, in column: String, rowIndex: Int) throws { + private static func refuseShortened(_ cell: PluginCellValue, in column: String, rowIndex: Int) throws(PluginRowWriteRefusal) { guard case let .text(text) = cell, JSONTruncation.isIncompleteStructure(text) else { return } throw PluginRowWriteRefusal( rowIndex: rowIndex, @@ -161,6 +162,16 @@ public enum SurrealStatementGenerator { // MARK: - Helpers + private static func reservedColumnReason(_ column: String) -> String { + guard column != SurrealInfoParser.recordIdColumn else { + return String(localized: "A record's id cannot be edited.") + } + let format = String( + localized: "'%@' cannot be saved from the grid. SurrealDB ignores it on a relation, so delete the relation and RELATE it again. Otherwise, UPDATE it in the editor." + ) + return String(format: format, column) + } + private static func recordId( table: String, columns: [String], diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 0535a15bc..a36312b5f 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -184770,6 +184770,18 @@ }, "Next cursor: %@. The scan has not finished." : { + }, + "A raw SurrealDB filter must be one condition that only reads. Run anything else in the SurrealQL editor." : { + + }, + "SurrealDB cannot filter rows with %@." : { + + }, + "A record's id cannot be edited." : { + + }, + "'%@' cannot be saved from the grid. SurrealDB ignores it on a relation, so delete the relation and RELATE it again. Otherwise, UPDATE it in the editor." : { + } }, "version" : "1.1" diff --git a/TableProTests/Plugins/SurrealDBDriverTests.swift b/TableProTests/Plugins/SurrealDBDriverTests.swift index 8b5077e2e..8529412a4 100644 --- a/TableProTests/Plugins/SurrealDBDriverTests.swift +++ b/TableProTests/Plugins/SurrealDBDriverTests.swift @@ -7,6 +7,8 @@ import Foundation import TableProPluginKit import Testing +@testable import TablePro + struct SurrealQLTests { @Test("Identifiers are backtick-quoted only when they need it") func identifiers() { @@ -97,7 +99,7 @@ struct SurrealQueryBuilderTests { let hostile = "x'; REMOVE TABLE person; --" let query = SurrealQueryBuilder.filtered( table: "person", scope: scope, - filters: [(column: "name", op: "=", value: hostile)], + filters: [PluginQueryFilter(column: "name", op: "=", value: hostile)], logicMode: "and", sortColumns: [], limit: 10, offset: 0 ) #expect(query.contains("name = 'x\\'; REMOVE TABLE person; --'")) @@ -105,48 +107,48 @@ struct SurrealQueryBuilderTests { } @Test("Filter operators map onto SurrealQL") - func operators() { - func clause(_ op: String, _ value: String) -> String { - SurrealQueryBuilder.whereClause( - filters: [(column: "c", op: op, value: value)], logicMode: "and" + func operators() throws { + func clause(_ op: String, _ value: String) throws -> String { + try SurrealQueryBuilder.whereClause( + filters: [PluginQueryFilter(column: "c", op: op, value: value)], logicMode: "and" ) ?? "" } - #expect(clause("=", "5") == "c = 5") - #expect(clause(">", "5") == "c > 5") - #expect(clause("=", "abc") == "c = 'abc'") - #expect(clause("=", "true") == "c = true") - #expect(clause("IS NULL", "") == "(c = NONE OR c = NULL)") - #expect(clause("CONTAINS", "x") == "string::contains( c, 'x')") - #expect(clause("STARTS WITH", "x") == "string::starts_with( c, 'x')") - #expect(clause("IN", "1, 2") == "c INSIDE [1, 2]") + #expect(try clause("=", "5") == "c = 5") + #expect(try clause(">", "5") == "c > 5") + #expect(try clause("=", "abc") == "c = 'abc'") + #expect(try clause("=", "true") == "c = true") + #expect(try clause("IS NULL", "") == "(c = NONE OR c = NULL)") + #expect(try clause("CONTAINS", "x") == "string::contains( c, 'x')") + #expect(try clause("STARTS WITH", "x") == "string::starts_with( c, 'x')") + #expect(try clause("IN", "1, 2") == "c INSIDE [1, 2]") } @Test("Only a real table:id shape becomes a record literal; look-alikes stay strings") - func literalRecordDisambiguation() { - func clause(_ value: String) -> String { - SurrealQueryBuilder.whereClause( - filters: [(column: "c", op: "=", value: value)], logicMode: "and" + func literalRecordDisambiguation() throws { + func clause(_ value: String) throws -> String { + try SurrealQueryBuilder.whereClause( + filters: [PluginQueryFilter(column: "c", op: "=", value: value)], logicMode: "and" ) ?? "" } - #expect(clause("person:tobie") == "c = person:tobie") - #expect(clause("12:30") == "c = '12:30'", "a time-shaped value is a string, not a record") - #expect(clause("1e5") == "c = 1e5") - #expect(clause("null") == "c = NULL") - #expect(clause("none") == "c = NONE") - #expect(clause("plain") == "c = 'plain'") + #expect(try clause("person:tobie") == "c = person:tobie") + #expect(try clause("12:30") == "c = '12:30'", "a time-shaped value is a string, not a record") + #expect(try clause("1e5") == "c = 1e5") + #expect(try clause("null") == "c = NULL") + #expect(try clause("none") == "c = NONE") + #expect(try clause("plain") == "c = 'plain'") } @Test("A hostile filter value cannot escape its literal in any branch") - func literalBranchesContainPayloads() { - func clause(_ value: String) -> String { - SurrealQueryBuilder.whereClause( - filters: [(column: "c", op: "=", value: value)], logicMode: "and" + func literalBranchesContainPayloads() throws { + func clause(_ value: String) throws -> String { + try SurrealQueryBuilder.whereClause( + filters: [PluginQueryFilter(column: "c", op: "=", value: value)], logicMode: "and" ) ?? "" } // String branch: the closing quote is escaped. - #expect(clause("x'; REMOVE TABLE person; --") == "c = 'x\\'; REMOVE TABLE person; --'") + #expect(try clause("x'; REMOVE TABLE person; --") == "c = 'x\\'; REMOVE TABLE person; --'") // Record branch: the id part is backtick-quoted, so ; stays inside the identifier. - let recordish = clause("person:a;REMOVE") + let recordish = try clause("person:a;REMOVE") #expect(recordish.hasPrefix("c = person:`") && recordish.hasSuffix("`")) } @@ -155,6 +157,182 @@ struct SurrealQueryBuilderTests { let query = SurrealQueryBuilder.count(table: "person", scope: scope, filters: [], logicMode: "and") #expect(query.contains("SELECT count() AS total FROM person GROUP ALL;")) } + + private func clause( + _ filter: TableFilter, + kinds: [String: PluginColumnKind] = [:] + ) throws -> String? { + try SurrealQueryBuilder.whereClause( + filters: [filter.asPluginQueryFilter], logicMode: "and", columnKinds: kinds + ) + } + + @Test("IS EMPTY matches a missing, null or empty-string field") + func isEmpty() throws { + let filter = TableFilter(columnName: "name", filterOperator: .isEmpty) + #expect(try clause(filter) == "(name = NONE OR name = NULL OR name = '')") + } + + @Test("IS NOT EMPTY keeps the fields that hold a value, not the empty ones") + func isNotEmpty() throws { + let filter = TableFilter(columnName: "name", filterOperator: .isNotEmpty) + #expect(try clause(filter) == "(name != NONE AND name != NULL AND name != '')") + } + + @Test("REGEX matches the pattern and passes over a missing or null field") + func regex() throws { + let filter = TableFilter(columnName: "name", filterOperator: .regex, value: "^A") + #expect(try clause(filter) == "(name != NONE AND name != NULL AND string::matches( name, '^A'))") + } + + @Test("The raw filter row is spliced in as a SurrealQL condition") + func rawRow() throws { + let filter = TableFilter(columnName: TableFilter.rawSQLColumn, rawSQL: "age > 10") + #expect(try clause(filter) == "(age > 10)") + } + + @Test("A raw condition that only reads keeps its strings, identifiers and function calls") + func rawConditionThatReads() throws { + let conditions = [ + "name = 'DELETE me' AND `update` = 1", + "array::len(array::remove([1, 2], 0)) = 1 OR ->likes->person CONTAINS person:\u{27E8}update\u{27E9}", + "(age > 10 AND age < 20) OR $session.update = NONE", + "age IN 10..20 AND string::lowercase(name) = 'a'" + ] + for text in conditions { + let filter = TableFilter(columnName: TableFilter.rawSQLColumn, rawSQL: text) + #expect(try clause(filter) == "(" + text + ")") + } + } + + @Test("A raw condition that could write, end the statement or hide the rest of it is refused") + func rawConditionThatWrites() { + let conditions = [ + "age > 1 OR (CREATE probe) = []", + "true OR { DELETE person }", + "age > 1 OR (upsert probe:one) = []", + "fn::wipe() = true", + "fn ::wipe() = true", + "fn\n::wipe() = true", + "http::post('https://example.com') = NONE", + "http ::post('https://example.com') = NONE", + "file::delete(bucket:/a.txt) = NONE", + "NONE ?:UPSERT probe:one SET n += 1", + "{a:UPSERT probe:one}.a != NONE", + "age > 1 ?:DELETE person:z", + "age IN 1..UPSERT probe:one SET n += 1", + "age > 1; REMOVE TABLE person", + "age > 1 -- the rest", + "age > 1 /* note */", + "age > 1 # note", + "age > 1) OR (true", + "name = 'unterminated" + ] + for text in conditions { + let filter = TableFilter(columnName: TableFilter.rawSQLColumn, rawSQL: text) + #expect(throws: SurrealFilterRefusal.rawConditionNotReadOnly, "\(text)") { + try clause(filter) + } + } + } + + @Test("A record id named after a writing keyword is refused unless its id is bracketed") + func rawConditionWithKeywordRecordId() throws { + let bare = TableFilter(columnName: TableFilter.rawSQLColumn, rawSQL: "id = person:update") + #expect(throws: SurrealFilterRefusal.rawConditionNotReadOnly) { + try clause(bare) + } + let bracketed = TableFilter(columnName: TableFilter.rawSQLColumn, rawSQL: "id = person:\u{27E8}update\u{27E9}") + #expect(try clause(bracketed) == "(id = person:\u{27E8}update\u{27E9})") + } + + @Test("A refused raw condition never reaches the server as SurrealQL") + func refusedRawConditionThrowsOnTheServer() { + let raw = TableFilter(columnName: TableFilter.rawSQLColumn, rawSQL: "true OR (DELETE person) = []") + let query = SurrealQueryBuilder.filtered( + table: "person", scope: scope, filters: [raw.asPluginQueryFilter], + logicMode: "and", sortColumns: [], limit: 10, offset: 0 + ) + #expect(query.hasPrefix("USE NS ns DB db;\nTHROW '")) + #expect(!query.contains("DELETE")) + } + + @Test("BETWEEN from the filter bar's own encoding uses both bounds") + func betweenFromAppEncoding() throws { + let filter = TableFilter(columnName: "age", filterOperator: .between, value: "10", secondValue: "20") + #expect(filter.asPluginQueryFilter.value == "10,20") + #expect(try clause(filter, kinds: ["age": .integer]) == "(age >= 10 AND age <= 20)") + } + + @Test("BETWEEN keeps a comma that belongs to a bound") + func betweenWithCommaInBound() throws { + let filter = TableFilter(columnName: "name", filterOperator: .between, value: "Smith, John", secondValue: "Zed") + #expect(try clause(filter, kinds: ["name": .text]) == "(name >= 'Smith, John' AND name <= 'Zed')") + } + + @Test("BETWEEN takes the lower bound off by scalars, so a bound ending in a prepend mark keeps its text") + func betweenWithPrependScalarBeforeTheSeparator() throws { + let filter = TableFilter(columnName: "code", filterOperator: .between, value: "A\u{0600}", secondValue: "B") + #expect(try clause(filter, kinds: ["code": .text]) == "(code >= 'A\u{0600}' AND code <= 'B')") + } + + @Test("BETWEEN without a separate upper bound splits the joined value") + func betweenFromJoinedValue() throws { + let clause = try SurrealQueryBuilder.whereClause( + filters: [PluginQueryFilter(column: "age", op: "BETWEEN", value: "10,20")], logicMode: "and" + ) + #expect(clause == "(age >= 10 AND age <= 20)") + } + + @Test("BETWEEN without a separate upper bound splits the joined value by scalars") + func betweenFromJoinedValueWithPrependScalar() throws { + let clause = try SurrealQueryBuilder.whereClause( + filters: [PluginQueryFilter(column: "code", op: "BETWEEN", value: "A\u{0600},B")], + logicMode: "and", + columnKinds: ["code": .text] + ) + #expect(clause == "(code >= 'A\u{0600}' AND code <= 'B')") + } + + @Test("BETWEEN with one bound is refused") + func betweenWithOneBound() { + #expect(throws: SurrealFilterRefusal.incompleteRange) { + try SurrealQueryBuilder.whereClause( + filters: [PluginQueryFilter(column: "age", op: "BETWEEN", value: "10")], logicMode: "and" + ) + } + } + + @Test("Every operator the filter bar offers has a condition of its own, never an equality") + func everyFilterBarOperatorIsMapped() throws { + for filterOperator in FilterOperator.allCases where filterOperator != .equal { + let filter = TableFilter(columnName: "c", filterOperator: filterOperator, value: "1", secondValue: "2") + let condition = try clause(filter) + #expect(condition != "c = 1", "\(filterOperator.rawValue) fell back to an equality") + } + } + + @Test("An operator SurrealDB has no condition for is refused") + func unknownOperatorRefused() { + #expect(throws: SurrealFilterRefusal.unsupportedOperator("SOUNDS LIKE")) { + try SurrealQueryBuilder.whereClause( + filters: [PluginQueryFilter(column: "name", op: "SOUNDS LIKE", value: "x")], logicMode: "and" + ) + } + } + + @Test("A refused filter runs as a THROW, so the grid reports it instead of showing no rows") + func refusedFilterThrowsOnTheServer() { + let filters = [PluginQueryFilter(column: "name", op: "SOUNDS LIKE", value: "x")] + let refusal = "USE NS ns DB db;\nTHROW 'SurrealDB cannot filter rows with SOUNDS LIKE.';" + let browse = SurrealQueryBuilder.filtered( + table: "person", scope: scope, filters: filters, + logicMode: "and", sortColumns: [], limit: 10, offset: 0 + ) + let count = SurrealQueryBuilder.count(table: "person", scope: scope, filters: filters, logicMode: "and") + #expect(browse == refusal) + #expect(count == refusal) + } } struct SurrealFieldKindTests { @@ -368,6 +546,21 @@ struct SurrealStatementGeneratorTests { SurrealCellCoder.value(from: cell) } + private func writes( + table: String = "person", + columns: [String]? = nil, + changes: [PluginRowChange] = [], + insertedRowData: [Int: [PluginCellValue]] = [:], + deletedRowIndices: Set = [], + insertedRowIndices: Set = [] + ) throws -> [PluginRowWrite] { + try SurrealStatementGenerator.rowWrites( + table: table, scope: scope, columns: columns ?? self.columns, kinds: kinds, + changes: changes, insertedRowData: insertedRowData, + deletedRowIndices: deletedRowIndices, insertedRowIndices: insertedRowIndices + ) + } + @Test("An update sets only the changed cells and never replaces the record") func update() throws { let change = PluginRowChange( @@ -376,20 +569,17 @@ struct SurrealStatementGeneratorTests { cellChanges: [(columnIndex: 2, columnName: "age", oldValue: .text("30"), newValue: .text("31"))], originalRow: [.text("person:alice"), .text("Alice"), .text("30")] ) - let writes = try SurrealStatementGenerator.rowWrites( - table: "person", scope: scope, columns: columns, kinds: kinds, - changes: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] - ) - let statement = try #require(writes.first) + let write = try #require(try writes(changes: [change]).first) - #expect(statement.statement.contains("UPDATE $p0 SET age = $p1;")) - #expect(!statement.statement.contains("CONTENT")) - #expect(!statement.statement.contains("MERGE")) - #expect(statement.statement.contains("USE NS ns DB db;")) + #expect(write.statement.contains("UPDATE $p0 SET age = $p1;")) + #expect(!write.statement.contains("CONTENT")) + #expect(!write.statement.contains("MERGE")) + #expect(write.statement.contains("USE NS ns DB db;")) + #expect(write.rowIndices == [0]) - #expect(decoded(statement.parameters[0]) + #expect(decoded(write.parameters[0]) == .recordId(SurrealRecordID(table: "person", id: .string("alice")))) - #expect(decoded(statement.parameters[1]) == .int(31), "an int column must bind as an int, not a string") + #expect(decoded(write.parameters[1]) == .int(31), "an int column must bind as an int, not a string") } @Test("The record id is bound, never interpolated") @@ -400,37 +590,28 @@ struct SurrealStatementGeneratorTests { cellChanges: [(columnIndex: 1, columnName: "name", oldValue: .text("a"), newValue: .text("'; REMOVE TABLE person; --"))], originalRow: [.text("person:alice"), .text("a"), .null] ) - let writes = try SurrealStatementGenerator.rowWrites( - table: "person", scope: scope, columns: columns, kinds: kinds, - changes: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] - ) - let statement = try #require(writes.first) - #expect(!statement.statement.contains("REMOVE TABLE")) - #expect(decoded(statement.parameters[1]) == .string("'; REMOVE TABLE person; --")) + let write = try #require(try writes(changes: [change]).first) + #expect(!write.statement.contains("REMOVE TABLE")) + #expect(decoded(write.parameters[1]) == .string("'; REMOVE TABLE person; --")) } @Test("An insert omits a blank id so the server mints one") func insert() throws { - let writes = try SurrealStatementGenerator.rowWrites( - table: "person", scope: scope, columns: columns, kinds: kinds, - changes: [], insertedRowData: [0: [.text(""), .text("Carol"), .text("22")]], - deletedRowIndices: [], insertedRowIndices: [0] - ) - let statement = try #require(writes.first) - #expect(statement.statement.contains("CREATE person SET name = $p0, age = $p1;")) - #expect(decoded(statement.parameters[1]) == .int(22)) + let write = try #require(try writes( + insertedRowData: [0: [.text(""), .text("Carol"), .text("22")]], insertedRowIndices: [0] + ).first) + #expect(write.statement.contains("CREATE person SET name = $p0, age = $p1;")) + #expect(decoded(write.parameters[1]) == .int(22)) + #expect(write.rowIndices == [0]) } @Test("An insert with an explicit id binds it as a record id") func insertWithId() throws { - let writes = try SurrealStatementGenerator.rowWrites( - table: "person", scope: scope, columns: columns, kinds: kinds, - changes: [], insertedRowData: [0: [.text("person:carol"), .text("Carol"), .null]], - deletedRowIndices: [], insertedRowIndices: [0] - ) - let statement = try #require(writes.first) - #expect(statement.statement.contains("CREATE $p0 SET name = $p1;")) - #expect(decoded(statement.parameters[0]) + let write = try #require(try writes( + insertedRowData: [0: [.text("person:carol"), .text("Carol"), .null]], insertedRowIndices: [0] + ).first) + #expect(write.statement.contains("CREATE $p0 SET name = $p1;")) + #expect(decoded(write.parameters[0]) == .recordId(SurrealRecordID(table: "person", id: .string("carol")))) } @@ -440,26 +621,41 @@ struct SurrealStatementGeneratorTests { rowIndex: 0, type: .delete, cellChanges: [], originalRow: [.text("person:alice"), .text("Alice"), .text("30")] ) - let writes = try SurrealStatementGenerator.rowWrites( - table: "person", scope: scope, columns: columns, kinds: kinds, - changes: [change], insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: [] + let write = try #require(try writes(changes: [change], deletedRowIndices: [0]).first) + #expect(write.statement.contains("DELETE $p0;")) + #expect(write.parameters.count == 1) + #expect(write.rowIndices == [0]) + } + + @Test("Each write names the row it carries out") + func eachWriteNamesItsRow() throws { + let update = PluginRowChange( + rowIndex: 3, + type: .update, + cellChanges: [(columnIndex: 1, columnName: "name", oldValue: .text("A"), newValue: .text("B"))], + originalRow: [.text("person:a"), .text("A"), .null] + ) + let delete = PluginRowChange( + rowIndex: 5, type: .delete, cellChanges: [], + originalRow: [.text("person:b"), .text("B"), .null] + ) + let all = try writes( + changes: [update, delete], + insertedRowData: [7: [.text(""), .text("C"), .null]], + deletedRowIndices: [5], + insertedRowIndices: [7] ) - let statement = try #require(writes.first) - #expect(statement.statement.contains("DELETE $p0;")) - #expect(statement.parameters.count == 1) + #expect(all.map(\.rowIndices) == [[3], [7], [5]]) } @Test("The auto-id marker on insert lets the server mint the id") func autoDefaultInsertId() throws { - let writes = try SurrealStatementGenerator.rowWrites( - table: "person", scope: scope, columns: columns, kinds: kinds, - changes: [], insertedRowData: [0: [.text("__DEFAULT__"), .text("Carol"), .text("22")]], - deletedRowIndices: [], insertedRowIndices: [0] - ) - let statement = try #require(writes.first) - #expect(statement.statement.contains("CREATE person SET")) - #expect(!statement.statement.contains("__DEFAULT__")) - #expect(!statement.statement.contains("person:__DEFAULT__")) + let write = try #require(try writes( + insertedRowData: [0: [.text("__DEFAULT__"), .text("Carol"), .text("22")]], insertedRowIndices: [0] + ).first) + #expect(write.statement.contains("CREATE person SET")) + #expect(!write.statement.contains("__DEFAULT__")) + #expect(!write.statement.contains("person:__DEFAULT__")) } @Test("The auto-id marker on an updated field is skipped, never written literally") @@ -470,48 +666,46 @@ struct SurrealStatementGeneratorTests { cellChanges: [(columnIndex: 1, columnName: "name", oldValue: .text("Alice"), newValue: .text("__DEFAULT__"))], originalRow: [.text("person:alice"), .text("Alice"), .text("30")] ) - let writes = try SurrealStatementGenerator.rowWrites( - table: "person", scope: scope, columns: columns, kinds: kinds, - changes: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] - ) - #expect(writes.isEmpty, "an all-default update produces no statement, not a literal write") + #expect(try writes(changes: [change]).isEmpty, "an all-default update produces no statement, not a literal write") } - @Test("The id column is never written") - func immutableId() throws { + @Test("An update with no cell changes has nothing to write and is not refused") + func emptyUpdate() throws { + let probe = PluginRowChange(rowIndex: 0, type: .update, cellChanges: [], originalRow: nil) + #expect(try writes(changes: [probe]).isEmpty) + } + + @Test("An edit to the id column is refused, never dropped") + func immutableId() { let change = PluginRowChange( - rowIndex: 0, + rowIndex: 4, type: .update, cellChanges: [(columnIndex: 0, columnName: "id", oldValue: .text("person:a"), newValue: .text("person:b"))], originalRow: [.text("person:a"), .text("Alice"), .null] ) - let writes = try SurrealStatementGenerator.rowWrites( - table: "person", scope: scope, columns: columns, kinds: kinds, - changes: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] - ) - #expect(writes.isEmpty) + #expect(throws: PluginRowWriteRefusal(rowIndex: 4, reason: "A record's id cannot be edited.")) { + try writes(changes: [change]) + } } - @Test("Each statement names the change it writes") - func writesNameTheirChange() throws { - let original: [PluginCellValue] = [.text("person:alice"), .text("Alice"), .text("30")] - let writes = try SurrealStatementGenerator.rowWrites( - table: "person", scope: scope, columns: columns, kinds: kinds, - changes: [ - PluginRowChange( - rowIndex: 0, - type: .update, - cellChanges: [(columnIndex: 2, columnName: "age", oldValue: .text("30"), newValue: .text("31"))], - originalRow: original - ), - PluginRowChange(rowIndex: 1, type: .insert, cellChanges: [], originalRow: nil), - PluginRowChange(rowIndex: 2, type: .delete, cellChanges: [], originalRow: original), + @Test("An edit to in or out beside another field is refused, never saved without it", arguments: ["in", "out"]) + func edgeEndpointEditIsRefused(column: String) throws { + let edgeColumns = ["id", "in", "out", "weight"] + let columnIndex = try #require(edgeColumns.firstIndex(of: column)) + let change = PluginRowChange( + rowIndex: 2, + type: .update, + cellChanges: [ + (columnIndex: columnIndex, columnName: column, oldValue: .text("person:a"), newValue: .text("person:c")), + (columnIndex: 3, columnName: "weight", oldValue: .text("1"), newValue: .text("2")) ], - insertedRowData: [1: [.text(""), .text("Carol"), .text("22")]], - deletedRowIndices: [2], - insertedRowIndices: [1] + originalRow: [.text("likes:x"), .text("person:a"), .text("person:b"), .text("1")] ) - #expect(writes.map(\.rowIndices) == [[0], [1], [2]]) + let error = #expect(throws: PluginRowWriteRefusal.self) { + try writes(table: "likes", columns: edgeColumns, changes: [change]) + } + #expect(error?.rowIndex == 2) + #expect(error?.reason.contains("'\(column)'") == true) } private let documentColumns = ["id", "tags", "meta"] diff --git a/docs/databases/surrealdb.mdx b/docs/databases/surrealdb.mdx index 24637508e..81f63fb81 100644 --- a/docs/databases/surrealdb.mdx +++ b/docs/databases/surrealdb.mdx @@ -66,6 +66,18 @@ Changing a cell runs an `UPDATE` that sets only the fields you touched, leaving - On a `RELATION` table, `in` and `out` sit next to `id`. - A `SCHEMALESS` table has no declared fields, so its columns come from the rows fetched: a field only some records carry still gets a column, and the rest show an empty cell. +## Filters + +Each filter bar operator becomes SurrealQL. **matches regex** runs `string::matches` and passes over records where the field is missing or NULL. **between** includes both bounds. **is empty** matches a missing field, NULL, or `''`. + +A **Raw SQL** row takes one SurrealQL condition, placed in parentheses in the `WHERE` clause: + +```sql +->likes->person CONTAINS person:alice +``` + +The condition must only read. One that holds a writing statement such as `CREATE`, `UPDATE` or `DELETE`, a `;`, a comment, or a call into `fn::`, `http::`, `api::`, `file::` or `sequence::` is refused, and the grid shows the error in place of rows. Run it in the editor instead. A record id named after one of those statements needs angle brackets: `person:⟨update⟩`. + ## SurrealQL The editor runs whole SurrealQL scripts, with completions for the common statements: @@ -88,7 +100,7 @@ The Explain button offers **Explain** and **Explain Full**, which rerun the curr ## Limitations - The structure editor is read-only. Create and change fields and indexes with `DEFINE FIELD` and `DEFINE INDEX` in the editor. -- Editing `in` or `out` on a `RELATION` table does nothing: those fields are dropped from the generated `UPDATE`. Rewire a relation with `RELATE` or an explicit `UPDATE`. +- An edit to `in` or `out` cannot be saved from the grid, on any table: the save stops, names the field, and keeps every edit pending. An `UPDATE` does not move a relation either, so delete it and `RELATE` the new pair. On any other record, change the field with an `UPDATE` in the editor. - Multi-request transactions are not available over HTTP. Send `BEGIN TRANSACTION; … COMMIT TRANSACTION;` as one editor query instead. - Live queries (`LIVE SELECT`) are not supported. Rerun the query to see new records. - A CA certificate file set in the **SSL/TLS** pane is ignored; verification goes through the system trust store, so install a private CA there. diff --git a/project.yml b/project.yml index b70891ebb..2a51ca535 100644 --- a/project.yml +++ b/project.yml @@ -845,6 +845,7 @@ targets: - Plugins/SurrealDBDriverPlugin/SurrealInfoParser.swift - Plugins/SurrealDBDriverPlugin/SurrealQL.swift - Plugins/SurrealDBDriverPlugin/SurrealQueryBuilder.swift + - Plugins/SurrealDBDriverPlugin/SurrealRawCondition.swift - Plugins/SurrealDBDriverPlugin/SurrealRowFlattener.swift - Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift - Plugins/SurrealDBDriverPlugin/SurrealValue+Display.swift