sql.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482
  1. # testing/fixtures/sql.py
  2. # Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of SQLAlchemy and is released under
  6. # the MIT License: https://www.opensource.org/licenses/mit-license.php
  7. # mypy: ignore-errors
  8. from __future__ import annotations
  9. import itertools
  10. import random
  11. import re
  12. import sqlalchemy as sa
  13. from .base import TestBase
  14. from .. import config
  15. from .. import mock
  16. from .. import provision
  17. from ..assertions import eq_
  18. from ..assertions import ne_
  19. from ..util import adict
  20. from ..util import drop_all_tables_from_metadata
  21. from ... import event
  22. from ... import util
  23. from ...schema import sort_tables_and_constraints
  24. from ...sql import visitors
  25. from ...sql.elements import ClauseElement
  26. class TablesTest(TestBase):
  27. # 'once', None
  28. run_setup_bind = "once"
  29. # 'once', 'each', None
  30. run_define_tables = "once"
  31. # 'once', 'each', None
  32. run_create_tables = "once"
  33. # 'once', 'each', None
  34. run_inserts = "each"
  35. # 'each', None
  36. run_deletes = "each"
  37. # 'once', None
  38. run_dispose_bind = None
  39. bind = None
  40. _tables_metadata = None
  41. tables = None
  42. other = None
  43. sequences = None
  44. @config.fixture(autouse=True, scope="class")
  45. def _setup_tables_test_class(self):
  46. cls = self.__class__
  47. cls._init_class()
  48. cls._setup_once_tables()
  49. cls._setup_once_inserts()
  50. yield
  51. cls._teardown_once_metadata_bind()
  52. @config.fixture(autouse=True, scope="function")
  53. def _setup_tables_test_instance(self):
  54. self._setup_each_tables()
  55. self._setup_each_inserts()
  56. yield
  57. self._teardown_each_tables()
  58. @property
  59. def tables_test_metadata(self):
  60. return self._tables_metadata
  61. @classmethod
  62. def _init_class(cls):
  63. if cls.run_define_tables == "each":
  64. if cls.run_create_tables == "once":
  65. cls.run_create_tables = "each"
  66. assert cls.run_inserts in ("each", None)
  67. cls.other = adict()
  68. cls.tables = adict()
  69. cls.sequences = adict()
  70. cls.bind = cls.setup_bind()
  71. cls._tables_metadata = sa.MetaData()
  72. @classmethod
  73. def _setup_once_inserts(cls):
  74. if cls.run_inserts == "once":
  75. cls._load_fixtures()
  76. with cls.bind.begin() as conn:
  77. cls.insert_data(conn)
  78. @classmethod
  79. def _setup_once_tables(cls):
  80. if cls.run_define_tables == "once":
  81. cls.define_tables(cls._tables_metadata)
  82. if cls.run_create_tables == "once":
  83. cls._tables_metadata.create_all(cls.bind)
  84. cls.tables.update(cls._tables_metadata.tables)
  85. cls.sequences.update(cls._tables_metadata._sequences)
  86. def _setup_each_tables(self):
  87. if self.run_define_tables == "each":
  88. self.define_tables(self._tables_metadata)
  89. if self.run_create_tables == "each":
  90. self._tables_metadata.create_all(self.bind)
  91. self.tables.update(self._tables_metadata.tables)
  92. self.sequences.update(self._tables_metadata._sequences)
  93. elif self.run_create_tables == "each":
  94. self._tables_metadata.create_all(self.bind)
  95. def _setup_each_inserts(self):
  96. if self.run_inserts == "each":
  97. self._load_fixtures()
  98. with self.bind.begin() as conn:
  99. self.insert_data(conn)
  100. def _teardown_each_tables(self):
  101. if self.run_define_tables == "each":
  102. self.tables.clear()
  103. if self.run_create_tables == "each":
  104. drop_all_tables_from_metadata(self._tables_metadata, self.bind)
  105. self._tables_metadata.clear()
  106. elif self.run_create_tables == "each":
  107. drop_all_tables_from_metadata(self._tables_metadata, self.bind)
  108. # no need to run deletes if tables are recreated on setup
  109. if (
  110. self.run_define_tables != "each"
  111. and self.run_create_tables == "once"
  112. and self.run_deletes == "each"
  113. ):
  114. with self.bind.begin() as conn:
  115. provision.delete_from_all_tables(
  116. conn, config, self._tables_metadata
  117. )
  118. @classmethod
  119. def _teardown_once_metadata_bind(cls):
  120. if cls.run_create_tables:
  121. drop_all_tables_from_metadata(cls._tables_metadata, cls.bind)
  122. if cls.run_dispose_bind == "once":
  123. cls.dispose_bind(cls.bind)
  124. cls._tables_metadata.bind = None
  125. if cls.run_setup_bind is not None:
  126. cls.bind = None
  127. @classmethod
  128. def setup_bind(cls):
  129. return config.db
  130. @classmethod
  131. def dispose_bind(cls, bind):
  132. if hasattr(bind, "dispose"):
  133. bind.dispose()
  134. elif hasattr(bind, "close"):
  135. bind.close()
  136. @classmethod
  137. def define_tables(cls, metadata):
  138. pass
  139. @classmethod
  140. def fixtures(cls):
  141. return {}
  142. @classmethod
  143. def insert_data(cls, connection):
  144. pass
  145. def sql_count_(self, count, fn):
  146. self.assert_sql_count(self.bind, fn, count)
  147. def sql_eq_(self, callable_, statements):
  148. self.assert_sql(self.bind, callable_, statements)
  149. @classmethod
  150. def _load_fixtures(cls):
  151. """Insert rows as represented by the fixtures() method."""
  152. headers, rows = {}, {}
  153. for table, data in cls.fixtures().items():
  154. if len(data) < 2:
  155. continue
  156. if isinstance(table, str):
  157. table = cls.tables[table]
  158. headers[table] = data[0]
  159. rows[table] = data[1:]
  160. for table, fks in sort_tables_and_constraints(
  161. cls._tables_metadata.tables.values()
  162. ):
  163. if table is None:
  164. continue
  165. if table not in headers:
  166. continue
  167. with cls.bind.begin() as conn:
  168. conn.execute(
  169. table.insert(),
  170. [
  171. dict(zip(headers[table], column_values))
  172. for column_values in rows[table]
  173. ],
  174. )
  175. class NoCache:
  176. @config.fixture(autouse=True, scope="function")
  177. def _disable_cache(self):
  178. _cache = config.db._compiled_cache
  179. config.db._compiled_cache = None
  180. yield
  181. config.db._compiled_cache = _cache
  182. class RemovesEvents:
  183. @util.memoized_property
  184. def _event_fns(self):
  185. return set()
  186. def event_listen(self, target, name, fn, **kw):
  187. self._event_fns.add((target, name, fn))
  188. event.listen(target, name, fn, **kw)
  189. @config.fixture(autouse=True, scope="function")
  190. def _remove_events(self):
  191. yield
  192. for key in self._event_fns:
  193. event.remove(*key)
  194. class ComputedReflectionFixtureTest(TablesTest):
  195. run_inserts = run_deletes = None
  196. __backend__ = True
  197. __requires__ = ("computed_columns", "table_reflection")
  198. regexp = re.compile(r"[\[\]\(\)\s`'\"]*")
  199. def normalize(self, text):
  200. return self.regexp.sub("", text).lower()
  201. @classmethod
  202. def define_tables(cls, metadata):
  203. from ... import Integer
  204. from ... import testing
  205. from ...schema import Column
  206. from ...schema import Computed
  207. from ...schema import Table
  208. Table(
  209. "computed_default_table",
  210. metadata,
  211. Column("id", Integer, primary_key=True),
  212. Column("normal", Integer),
  213. Column("computed_col", Integer, Computed("normal + 42")),
  214. Column("with_default", Integer, server_default="42"),
  215. )
  216. t = Table(
  217. "computed_column_table",
  218. metadata,
  219. Column("id", Integer, primary_key=True),
  220. Column("normal", Integer),
  221. Column("computed_no_flag", Integer, Computed("normal + 42")),
  222. )
  223. if testing.requires.schemas.enabled:
  224. t2 = Table(
  225. "computed_column_table",
  226. metadata,
  227. Column("id", Integer, primary_key=True),
  228. Column("normal", Integer),
  229. Column("computed_no_flag", Integer, Computed("normal / 42")),
  230. schema=config.test_schema,
  231. )
  232. if testing.requires.computed_columns_virtual.enabled:
  233. t.append_column(
  234. Column(
  235. "computed_virtual",
  236. Integer,
  237. Computed("normal + 2", persisted=False),
  238. )
  239. )
  240. if testing.requires.schemas.enabled:
  241. t2.append_column(
  242. Column(
  243. "computed_virtual",
  244. Integer,
  245. Computed("normal / 2", persisted=False),
  246. )
  247. )
  248. if testing.requires.computed_columns_stored.enabled:
  249. t.append_column(
  250. Column(
  251. "computed_stored",
  252. Integer,
  253. Computed("normal - 42", persisted=True),
  254. )
  255. )
  256. if testing.requires.schemas.enabled:
  257. t2.append_column(
  258. Column(
  259. "computed_stored",
  260. Integer,
  261. Computed("normal * 42", persisted=True),
  262. )
  263. )
  264. class CacheKeyFixture:
  265. def _compare_equal(self, a, b, compare_values):
  266. a_key = a._generate_cache_key()
  267. b_key = b._generate_cache_key()
  268. if a_key is None:
  269. assert a._annotations.get("nocache")
  270. assert b_key is None
  271. else:
  272. eq_(a_key.key, b_key.key)
  273. eq_(hash(a_key.key), hash(b_key.key))
  274. for a_param, b_param in zip(a_key.bindparams, b_key.bindparams):
  275. assert a_param.compare(b_param, compare_values=compare_values)
  276. return a_key, b_key
  277. def _run_cache_key_fixture(self, fixture, compare_values):
  278. case_a = fixture()
  279. case_b = fixture()
  280. for a, b in itertools.combinations_with_replacement(
  281. range(len(case_a)), 2
  282. ):
  283. if a == b:
  284. a_key, b_key = self._compare_equal(
  285. case_a[a], case_b[b], compare_values
  286. )
  287. if a_key is None:
  288. continue
  289. else:
  290. a_key = case_a[a]._generate_cache_key()
  291. b_key = case_b[b]._generate_cache_key()
  292. if a_key is None or b_key is None:
  293. if a_key is None:
  294. assert case_a[a]._annotations.get("nocache")
  295. if b_key is None:
  296. assert case_b[b]._annotations.get("nocache")
  297. continue
  298. if a_key.key == b_key.key:
  299. for a_param, b_param in zip(
  300. a_key.bindparams, b_key.bindparams
  301. ):
  302. if not a_param.compare(
  303. b_param, compare_values=compare_values
  304. ):
  305. break
  306. else:
  307. # this fails unconditionally since we could not
  308. # find bound parameter values that differed.
  309. # Usually we intended to get two distinct keys here
  310. # so the failure will be more descriptive using the
  311. # ne_() assertion.
  312. ne_(a_key.key, b_key.key)
  313. else:
  314. ne_(a_key.key, b_key.key)
  315. # ClauseElement-specific test to ensure the cache key
  316. # collected all the bound parameters that aren't marked
  317. # as "literal execute"
  318. if isinstance(case_a[a], ClauseElement) and isinstance(
  319. case_b[b], ClauseElement
  320. ):
  321. assert_a_params = []
  322. assert_b_params = []
  323. for elem in visitors.iterate(case_a[a]):
  324. if elem.__visit_name__ == "bindparam":
  325. assert_a_params.append(elem)
  326. for elem in visitors.iterate(case_b[b]):
  327. if elem.__visit_name__ == "bindparam":
  328. assert_b_params.append(elem)
  329. # note we're asserting the order of the params as well as
  330. # if there are dupes or not. ordering has to be
  331. # deterministic and matches what a traversal would provide.
  332. eq_(
  333. sorted(a_key.bindparams, key=lambda b: b.key),
  334. sorted(
  335. util.unique_list(assert_a_params), key=lambda b: b.key
  336. ),
  337. )
  338. eq_(
  339. sorted(b_key.bindparams, key=lambda b: b.key),
  340. sorted(
  341. util.unique_list(assert_b_params), key=lambda b: b.key
  342. ),
  343. )
  344. def _run_cache_key_equal_fixture(self, fixture, compare_values):
  345. case_a = fixture()
  346. case_b = fixture()
  347. for a, b in itertools.combinations_with_replacement(
  348. range(len(case_a)), 2
  349. ):
  350. self._compare_equal(case_a[a], case_b[b], compare_values)
  351. def insertmanyvalues_fixture(
  352. connection, randomize_rows=False, warn_on_downgraded=False
  353. ):
  354. dialect = connection.dialect
  355. orig_dialect = dialect._deliver_insertmanyvalues_batches
  356. orig_conn = connection._exec_insertmany_context
  357. class RandomCursor:
  358. __slots__ = ("cursor",)
  359. def __init__(self, cursor):
  360. self.cursor = cursor
  361. # only this method is called by the deliver method.
  362. # by not having the other methods we assert that those aren't being
  363. # used
  364. @property
  365. def description(self):
  366. return self.cursor.description
  367. def fetchall(self):
  368. rows = self.cursor.fetchall()
  369. rows = list(rows)
  370. random.shuffle(rows)
  371. return rows
  372. def _deliver_insertmanyvalues_batches(
  373. connection,
  374. cursor,
  375. statement,
  376. parameters,
  377. generic_setinputsizes,
  378. context,
  379. ):
  380. if randomize_rows:
  381. cursor = RandomCursor(cursor)
  382. for batch in orig_dialect(
  383. connection,
  384. cursor,
  385. statement,
  386. parameters,
  387. generic_setinputsizes,
  388. context,
  389. ):
  390. if warn_on_downgraded and batch.is_downgraded:
  391. util.warn("Batches were downgraded for sorted INSERT")
  392. yield batch
  393. def _exec_insertmany_context(dialect, context):
  394. with mock.patch.object(
  395. dialect,
  396. "_deliver_insertmanyvalues_batches",
  397. new=_deliver_insertmanyvalues_batches,
  398. ):
  399. return orig_conn(dialect, context)
  400. connection._exec_insertmany_context = _exec_insertmany_context