json.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404
  1. # dialects/postgresql/json.py
  2. # Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of SQLAlchemy and is released under
  6. # the MIT License: https://www.opensource.org/licenses/mit-license.php
  7. from __future__ import annotations
  8. from typing import Any
  9. from typing import Callable
  10. from typing import List
  11. from typing import Optional
  12. from typing import TYPE_CHECKING
  13. from typing import Union
  14. from .array import ARRAY
  15. from .array import array as _pg_array
  16. from .operators import ASTEXT
  17. from .operators import CONTAINED_BY
  18. from .operators import CONTAINS
  19. from .operators import DELETE_PATH
  20. from .operators import HAS_ALL
  21. from .operators import HAS_ANY
  22. from .operators import HAS_KEY
  23. from .operators import JSONPATH_ASTEXT
  24. from .operators import PATH_EXISTS
  25. from .operators import PATH_MATCH
  26. from ... import types as sqltypes
  27. from ...sql import cast
  28. from ...sql._typing import _T
  29. if TYPE_CHECKING:
  30. from ...engine.interfaces import Dialect
  31. from ...sql.elements import ColumnElement
  32. from ...sql.operators import OperatorType
  33. from ...sql.type_api import _BindProcessorType
  34. from ...sql.type_api import _LiteralProcessorType
  35. from ...sql.type_api import TypeEngine
  36. __all__ = ("JSON", "JSONB")
  37. class JSONPathType(sqltypes.JSON.JSONPathType):
  38. def _processor(
  39. self, dialect: Dialect, super_proc: Optional[Callable[[Any], Any]]
  40. ) -> Callable[[Any], Any]:
  41. def process(value: Any) -> Any:
  42. if isinstance(value, str):
  43. # If it's already a string assume that it's in json path
  44. # format. This allows using cast with json paths literals
  45. return value
  46. elif value:
  47. # If it's already a string assume that it's in json path
  48. # format. This allows using cast with json paths literals
  49. value = "{%s}" % (", ".join(map(str, value)))
  50. else:
  51. value = "{}"
  52. if super_proc:
  53. value = super_proc(value)
  54. return value
  55. return process
  56. def bind_processor(self, dialect: Dialect) -> _BindProcessorType[Any]:
  57. return self._processor(dialect, self.string_bind_processor(dialect)) # type: ignore[return-value] # noqa: E501
  58. def literal_processor(
  59. self, dialect: Dialect
  60. ) -> _LiteralProcessorType[Any]:
  61. return self._processor(dialect, self.string_literal_processor(dialect)) # type: ignore[return-value] # noqa: E501
  62. class JSONPATH(JSONPathType):
  63. """JSON Path Type.
  64. This is usually required to cast literal values to json path when using
  65. json search like function, such as ``jsonb_path_query_array`` or
  66. ``jsonb_path_exists``::
  67. stmt = sa.select(
  68. sa.func.jsonb_path_query_array(
  69. table.c.jsonb_col, cast("$.address.id", JSONPATH)
  70. )
  71. )
  72. """
  73. __visit_name__ = "JSONPATH"
  74. class JSON(sqltypes.JSON):
  75. """Represent the PostgreSQL JSON type.
  76. :class:`_postgresql.JSON` is used automatically whenever the base
  77. :class:`_types.JSON` datatype is used against a PostgreSQL backend,
  78. however base :class:`_types.JSON` datatype does not provide Python
  79. accessors for PostgreSQL-specific comparison methods such as
  80. :meth:`_postgresql.JSON.Comparator.astext`; additionally, to use
  81. PostgreSQL ``JSONB``, the :class:`_postgresql.JSONB` datatype should
  82. be used explicitly.
  83. .. seealso::
  84. :class:`_types.JSON` - main documentation for the generic
  85. cross-platform JSON datatype.
  86. The operators provided by the PostgreSQL version of :class:`_types.JSON`
  87. include:
  88. * Index operations (the ``->`` operator)::
  89. data_table.c.data["some key"]
  90. data_table.c.data[5]
  91. * Index operations returning text
  92. (the ``->>`` operator)::
  93. data_table.c.data["some key"].astext == "some value"
  94. Note that equivalent functionality is available via the
  95. :attr:`.JSON.Comparator.as_string` accessor.
  96. * Index operations with CAST
  97. (equivalent to ``CAST(col ->> ['some key'] AS <type>)``)::
  98. data_table.c.data["some key"].astext.cast(Integer) == 5
  99. Note that equivalent functionality is available via the
  100. :attr:`.JSON.Comparator.as_integer` and similar accessors.
  101. * Path index operations (the ``#>`` operator)::
  102. data_table.c.data[("key_1", "key_2", 5, ..., "key_n")]
  103. * Path index operations returning text (the ``#>>`` operator)::
  104. data_table.c.data[
  105. ("key_1", "key_2", 5, ..., "key_n")
  106. ].astext == "some value"
  107. Index operations return an expression object whose type defaults to
  108. :class:`_types.JSON` by default,
  109. so that further JSON-oriented instructions
  110. may be called upon the result type.
  111. Custom serializers and deserializers are specified at the dialect level,
  112. that is using :func:`_sa.create_engine`. The reason for this is that when
  113. using psycopg2, the DBAPI only allows serializers at the per-cursor
  114. or per-connection level. E.g.::
  115. engine = create_engine(
  116. "postgresql+psycopg2://scott:tiger@localhost/test",
  117. json_serializer=my_serialize_fn,
  118. json_deserializer=my_deserialize_fn,
  119. )
  120. When using the psycopg2 dialect, the json_deserializer is registered
  121. against the database using ``psycopg2.extras.register_default_json``.
  122. .. seealso::
  123. :class:`_types.JSON` - Core level JSON type
  124. :class:`_postgresql.JSONB`
  125. """ # noqa
  126. render_bind_cast = True
  127. astext_type: TypeEngine[str] = sqltypes.Text()
  128. def __init__(
  129. self,
  130. none_as_null: bool = False,
  131. astext_type: Optional[TypeEngine[str]] = None,
  132. ):
  133. """Construct a :class:`_types.JSON` type.
  134. :param none_as_null: if True, persist the value ``None`` as a
  135. SQL NULL value, not the JSON encoding of ``null``. Note that
  136. when this flag is False, the :func:`.null` construct can still
  137. be used to persist a NULL value::
  138. from sqlalchemy import null
  139. conn.execute(table.insert(), {"data": null()})
  140. .. seealso::
  141. :attr:`_types.JSON.NULL`
  142. :param astext_type: the type to use for the
  143. :attr:`.JSON.Comparator.astext`
  144. accessor on indexed attributes. Defaults to :class:`_types.Text`.
  145. """
  146. super().__init__(none_as_null=none_as_null)
  147. if astext_type is not None:
  148. self.astext_type = astext_type
  149. class Comparator(sqltypes.JSON.Comparator[_T]):
  150. """Define comparison operations for :class:`_types.JSON`."""
  151. type: JSON
  152. @property
  153. def astext(self) -> ColumnElement[str]:
  154. """On an indexed expression, use the "astext" (e.g. "->>")
  155. conversion when rendered in SQL.
  156. E.g.::
  157. select(data_table.c.data["some key"].astext)
  158. .. seealso::
  159. :meth:`_expression.ColumnElement.cast`
  160. """
  161. if isinstance(self.expr.right.type, sqltypes.JSON.JSONPathType):
  162. return self.expr.left.operate( # type: ignore[no-any-return]
  163. JSONPATH_ASTEXT,
  164. self.expr.right,
  165. result_type=self.type.astext_type,
  166. )
  167. else:
  168. return self.expr.left.operate( # type: ignore[no-any-return]
  169. ASTEXT, self.expr.right, result_type=self.type.astext_type
  170. )
  171. comparator_factory = Comparator
  172. class JSONB(JSON):
  173. """Represent the PostgreSQL JSONB type.
  174. The :class:`_postgresql.JSONB` type stores arbitrary JSONB format data,
  175. e.g.::
  176. data_table = Table(
  177. "data_table",
  178. metadata,
  179. Column("id", Integer, primary_key=True),
  180. Column("data", JSONB),
  181. )
  182. with engine.connect() as conn:
  183. conn.execute(
  184. data_table.insert(), data={"key1": "value1", "key2": "value2"}
  185. )
  186. The :class:`_postgresql.JSONB` type includes all operations provided by
  187. :class:`_types.JSON`, including the same behaviors for indexing
  188. operations.
  189. It also adds additional operators specific to JSONB, including
  190. :meth:`.JSONB.Comparator.has_key`, :meth:`.JSONB.Comparator.has_all`,
  191. :meth:`.JSONB.Comparator.has_any`, :meth:`.JSONB.Comparator.contains`,
  192. :meth:`.JSONB.Comparator.contained_by`,
  193. :meth:`.JSONB.Comparator.delete_path`,
  194. :meth:`.JSONB.Comparator.path_exists` and
  195. :meth:`.JSONB.Comparator.path_match`.
  196. Like the :class:`_types.JSON` type, the :class:`_postgresql.JSONB`
  197. type does not detect
  198. in-place changes when used with the ORM, unless the
  199. :mod:`sqlalchemy.ext.mutable` extension is used.
  200. Custom serializers and deserializers
  201. are shared with the :class:`_types.JSON` class,
  202. using the ``json_serializer``
  203. and ``json_deserializer`` keyword arguments. These must be specified
  204. at the dialect level using :func:`_sa.create_engine`. When using
  205. psycopg2, the serializers are associated with the jsonb type using
  206. ``psycopg2.extras.register_default_jsonb`` on a per-connection basis,
  207. in the same way that ``psycopg2.extras.register_default_json`` is used
  208. to register these handlers with the json type.
  209. .. seealso::
  210. :class:`_types.JSON`
  211. .. warning::
  212. **For applications that have indexes against JSONB subscript
  213. expressions**
  214. SQLAlchemy 2.0.42 made a change in how the subscript operation for
  215. :class:`.JSONB` is rendered, from ``-> 'element'`` to ``['element']``,
  216. for PostgreSQL versions greater than 14. This change caused an
  217. unintended side effect for indexes that were created against
  218. expressions that use subscript notation, e.g.
  219. ``Index("ix_entity_json_ab_text", data["a"]["b"].astext)``. If these
  220. indexes were generated with the older syntax e.g. ``((entity.data ->
  221. 'a') ->> 'b')``, they will not be used by the PostgreSQL query planner
  222. when a query is made using SQLAlchemy 2.0.42 or higher on PostgreSQL
  223. versions 14 or higher. This occurs because the new text will resemble
  224. ``(entity.data['a'] ->> 'b')`` which will fail to produce the exact
  225. textual syntax match required by the PostgreSQL query planner.
  226. Therefore, for users upgrading to SQLAlchemy 2.0.42 or higher, existing
  227. indexes that were created against :class:`.JSONB` expressions that use
  228. subscripting would need to be dropped and re-created in order for them
  229. to work with the new query syntax, e.g. an expression like
  230. ``((entity.data -> 'a') ->> 'b')`` would become ``(entity.data['a'] ->>
  231. 'b')``.
  232. .. seealso::
  233. :ticket:`12868` - discussion of this issue
  234. """
  235. __visit_name__ = "JSONB"
  236. def coerce_compared_value(
  237. self, op: Optional[OperatorType], value: Any
  238. ) -> TypeEngine[Any]:
  239. if op in (PATH_MATCH, PATH_EXISTS):
  240. return JSON.JSONPathType()
  241. else:
  242. return super().coerce_compared_value(op, value)
  243. class Comparator(JSON.Comparator[_T]):
  244. """Define comparison operations for :class:`_types.JSON`."""
  245. type: JSONB
  246. def has_key(self, other: Any) -> ColumnElement[bool]:
  247. """Boolean expression. Test for presence of a key (equivalent of
  248. the ``?`` operator). Note that the key may be a SQLA expression.
  249. """
  250. return self.operate(HAS_KEY, other, result_type=sqltypes.Boolean)
  251. def has_all(self, other: Any) -> ColumnElement[bool]:
  252. """Boolean expression. Test for presence of all keys in jsonb
  253. (equivalent of the ``?&`` operator)
  254. """
  255. return self.operate(HAS_ALL, other, result_type=sqltypes.Boolean)
  256. def has_any(self, other: Any) -> ColumnElement[bool]:
  257. """Boolean expression. Test for presence of any key in jsonb
  258. (equivalent of the ``?|`` operator)
  259. """
  260. return self.operate(HAS_ANY, other, result_type=sqltypes.Boolean)
  261. def contains(self, other: Any, **kwargs: Any) -> ColumnElement[bool]:
  262. """Boolean expression. Test if keys (or array) are a superset
  263. of/contained the keys of the argument jsonb expression
  264. (equivalent of the ``@>`` operator).
  265. kwargs may be ignored by this operator but are required for API
  266. conformance.
  267. """
  268. return self.operate(CONTAINS, other, result_type=sqltypes.Boolean)
  269. def contained_by(self, other: Any) -> ColumnElement[bool]:
  270. """Boolean expression. Test if keys are a proper subset of the
  271. keys of the argument jsonb expression
  272. (equivalent of the ``<@`` operator).
  273. """
  274. return self.operate(
  275. CONTAINED_BY, other, result_type=sqltypes.Boolean
  276. )
  277. def delete_path(
  278. self, array: Union[List[str], _pg_array[str]]
  279. ) -> ColumnElement[JSONB]:
  280. """JSONB expression. Deletes field or array element specified in
  281. the argument array (equivalent of the ``#-`` operator).
  282. The input may be a list of strings that will be coerced to an
  283. ``ARRAY`` or an instance of :meth:`_postgres.array`.
  284. .. versionadded:: 2.0
  285. """
  286. if not isinstance(array, _pg_array):
  287. array = _pg_array(array)
  288. right_side = cast(array, ARRAY(sqltypes.TEXT))
  289. return self.operate(DELETE_PATH, right_side, result_type=JSONB)
  290. def path_exists(self, other: Any) -> ColumnElement[bool]:
  291. """Boolean expression. Test for presence of item given by the
  292. argument JSONPath expression (equivalent of the ``@?`` operator).
  293. .. versionadded:: 2.0
  294. """
  295. return self.operate(
  296. PATH_EXISTS, other, result_type=sqltypes.Boolean
  297. )
  298. def path_match(self, other: Any) -> ColumnElement[bool]:
  299. """Boolean expression. Test if JSONPath predicate given by the
  300. argument JSONPath expression matches
  301. (equivalent of the ``@@`` operator).
  302. Only the first item of the result is taken into account.
  303. .. versionadded:: 2.0
  304. """
  305. return self.operate(
  306. PATH_MATCH, other, result_type=sqltypes.Boolean
  307. )
  308. comparator_factory = Comparator