From 040b8180ecf0eae504a2f4def2770b0f83eed399 Mon Sep 17 00:00:00 2001 From: Amin Ghadersohi Date: Sat, 26 Sep 2026 07:14:19 +0000 Subject: [PATCH] Decode NUMERIC columns as Decimal without precision loss orjson decodes every JSON number to a float, so NUMERIC values lost the digits beyond double precision that CrateDB sends. When a response declares a NUMERIC column, or an array of them, decode the body a second time with Decimal floats and replace only those columns. Other columns keep their float values, and responses without NUMERIC columns are decoded once, as before. Fixes #826 --- CHANGES.rst | 8 +++++++ docs/data-types.rst | 7 ++++++ src/crate/client/http.py | 49 ++++++++++++++++++++++++++++++++++++++- tests/client/test_http.py | 38 ++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5ea4df71..ded6db9f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,14 @@ Changes for crate ================= +Unreleased +========== +- Fixed ``NUMERIC`` column values losing digits beyond double precision when + reading. They are now decoded as ``Decimal``, including ``NUMERIC`` arrays. + Other columns keep their ``float`` values. See `#826`_. + +.. _#826: https://github.com/crate/crate-python/issues/826 + 2026/09/17 2.3.0 ================ - Added ``DefaultTypeConverter`` support that decodes ``DataType.UUID`` diff --git a/docs/data-types.rst b/docs/data-types.rst index 90fd9234..4b63becd 100644 --- a/docs/data-types.rst +++ b/docs/data-types.rst @@ -64,6 +64,13 @@ __ https://docs.python.org/3/library/stdtypes.html#dict __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#array __ https://docs.python.org/3/library/stdtypes.html#list +.. NOTE:: + + ``numeric`` values are returned as `Decimal`_, with every digit CrateDB + sends. Other numeric types use the mapping above. + +.. _Decimal: https://docs.python.org/3/library/decimal.html + When writing to CrateDB, the following conversions take place: ============= ==================================== diff --git a/src/crate/client/http.py b/src/crate/client/http.py index 7492ecf2..15fbde58 100644 --- a/src/crate/client/http.py +++ b/src/crate/client/http.py @@ -25,6 +25,7 @@ import gzip import heapq import io +import json import logging import os import re @@ -52,6 +53,7 @@ from urllib3.util.retry import Retry from verlib2 import Version +from crate.client.converter import DataType from crate.client.exceptions import ( BlobLocationNotFoundException, ConnectionError, @@ -241,7 +243,7 @@ def close(self): def _json_from_response(response): try: - return orjson.loads(response.data) + data = orjson.loads(response.data) except ValueError as ex: raise ProgrammingError( "Invalid server response of content-type '{}':\n{}".format( @@ -249,6 +251,51 @@ def _json_from_response(response): response.data.decode("utf-8"), ) ) from ex + return _exact_numeric_columns(data, response.data) + + +def _is_numeric_type(col_type) -> bool: + if isinstance(col_type, list): + return ( + len(col_type) == 2 + and col_type[0] == DataType.ARRAY.value + and _is_numeric_type(col_type[1]) + ) + return col_type == DataType.NUMERIC.value + + +def _to_decimal(value): + if isinstance(value, list): + return [_to_decimal(item) for item in value] + if isinstance(value, (int, Decimal)) and not isinstance(value, bool): + return Decimal(value) + return value + + +def _exact_numeric_columns(data, raw: bytes): + """ + Decode ``NUMERIC`` column values as ``Decimal``, without precision loss. + + CrateDB sends every digit of a ``NUMERIC`` value, but ``orjson`` decodes + each JSON number to a 64-bit float. Only when the response declares a + ``NUMERIC`` column (or an array of them), the body is decoded a second + time with ``Decimal`` floats, and only those columns are replaced. Other + columns, including ``DOUBLE``, keep their ``float`` values. + """ + if not isinstance(data, dict) or not data.get("rows"): + return data + columns = [ + index + for index, col_type in enumerate(data.get("col_types") or []) + if _is_numeric_type(col_type) + ] + if not columns: + return data + exact_rows = json.loads(raw, parse_float=Decimal)["rows"] + for row, exact_row in zip(data["rows"], exact_rows, strict=True): + for index in columns: + row[index] = _to_decimal(exact_row[index]) + return data def _blob_path(table, digest): diff --git a/tests/client/test_http.py b/tests/client/test_http.py index 3510de41..000f9864 100644 --- a/tests/client/test_http.py +++ b/tests/client/test_http.py @@ -27,6 +27,7 @@ import socket import time from base64 import b64decode +from decimal import Decimal from http.server import BaseHTTPRequestHandler from threading import Event, Thread from unittest.mock import MagicMock, patch @@ -872,3 +873,40 @@ def capturing(*_, **kwargs): Client(servers="localhost:4200").sql("SELECT 1") assert captured["headers"].get("Accept-Encoding") == "gzip, deflate" assert "Content-Encoding" not in captured["headers"] + + +def test_numeric_columns_decode_without_precision_loss(): + """ + NUMERIC values keep every digit; other columns keep their float values. + """ + response = fake_response(200) + response.data = ( + b'{"cols":["n","d","na","t"],"col_types":[22,6,[100,22],4],' + b'"rows":[[12345678901234567890.123456789012345678,0.1,[1.5,2,null],"x"],' + b'[null,1.0E300,null,"y"],[7,2,[],"z"]],"rowcount":3,"duration":1.0}' + ) + with patch(REQUEST_PATH, return_value=response): + client = Client(servers="localhost:4200") + rows = client.sql("SELECT n, d, na, t FROM t")["rows"] + assert rows[0] == [ + Decimal("12345678901234567890.123456789012345678"), + 0.1, + [Decimal("1.5"), Decimal(2), None], + "x", + ] + assert type(rows[0][0]) is Decimal + assert type(rows[0][1]) is float + assert rows[1] == [None, 1.0e300, None, "y"] + assert type(rows[1][1]) is float + assert rows[2] == [Decimal(7), 2, [], "z"] + assert type(rows[2][0]) is Decimal + assert type(rows[2][1]) is int + + +def test_responses_without_numeric_columns_unchanged(): + response = fake_response(200) + response.data = b'{"cols":["d"],"col_types":[6],"rows":[[0.5]]}' + with patch(REQUEST_PATH, return_value=response): + result = Client(servers="localhost:4200").sql("SELECT d FROM t") + assert result["rows"] == [[0.5]] + assert type(result["rows"][0][0]) is float