provision.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297
  1. # dialects/oracle/provision.py
  2. # Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of SQLAlchemy and is released under
  6. # the MIT License: https://www.opensource.org/licenses/mit-license.php
  7. # mypy: ignore-errors
  8. import time
  9. from ... import create_engine
  10. from ... import exc
  11. from ... import inspect
  12. from ...engine import url as sa_url
  13. from ...testing.provision import configure_follower
  14. from ...testing.provision import create_db
  15. from ...testing.provision import drop_all_schema_objects_post_tables
  16. from ...testing.provision import drop_all_schema_objects_pre_tables
  17. from ...testing.provision import drop_db
  18. from ...testing.provision import follower_url_from_main
  19. from ...testing.provision import generate_driver_url
  20. from ...testing.provision import is_preferred_driver
  21. from ...testing.provision import log
  22. from ...testing.provision import post_configure_engine
  23. from ...testing.provision import post_configure_testing_engine
  24. from ...testing.provision import run_reap_dbs
  25. from ...testing.provision import set_default_schema_on_connection
  26. from ...testing.provision import stop_test_class_outside_fixtures
  27. from ...testing.provision import temp_table_keyword_args
  28. from ...testing.provision import update_db_opts
  29. from ...testing.warnings import warn_test_suite
  30. @generate_driver_url.for_db("oracle")
  31. def _oracle_generate_driver_url(url, driver, query_str):
  32. backend = url.get_backend_name()
  33. new_url = url.set(
  34. drivername="%s+%s" % (backend, driver),
  35. )
  36. # use oracledb's retry feature, which is essential for oracle 23c
  37. # which otherwise frequently rejects connections under load
  38. # for cx_oracle we have a connect event instead
  39. if driver in ("oracledb", "oracledb_async"):
  40. # oracledb is even nice enough to convert from string to int
  41. # for these opts, apparently
  42. new_url = new_url.update_query_pairs(
  43. [("retry_count", "5"), ("retry_delay", "2")]
  44. )
  45. else:
  46. # remove these params for cx_oracle if we received an
  47. # already-modified URL
  48. new_url = new_url.difference_update_query(
  49. ["retry_count", "retry_delay"]
  50. )
  51. try:
  52. new_url.get_dialect()
  53. except exc.NoSuchModuleError:
  54. return None
  55. else:
  56. return new_url
  57. @create_db.for_db("oracle")
  58. def _oracle_create_db(cfg, eng, ident):
  59. # NOTE: make sure you've run "ALTER DATABASE default tablespace users" or
  60. # similar, so that the default tablespace is not "system"; reflection will
  61. # fail otherwise
  62. with eng.begin() as conn:
  63. conn.exec_driver_sql("create user %s identified by xe" % ident)
  64. conn.exec_driver_sql("create user %s_ts1 identified by xe" % ident)
  65. conn.exec_driver_sql("create user %s_ts2 identified by xe" % ident)
  66. conn.exec_driver_sql("grant dba to %s" % (ident,))
  67. conn.exec_driver_sql("grant unlimited tablespace to %s" % ident)
  68. conn.exec_driver_sql("grant unlimited tablespace to %s_ts1" % ident)
  69. conn.exec_driver_sql("grant unlimited tablespace to %s_ts2" % ident)
  70. # these are needed to create materialized views
  71. conn.exec_driver_sql("grant create table to %s" % ident)
  72. conn.exec_driver_sql("grant create table to %s_ts1" % ident)
  73. conn.exec_driver_sql("grant create table to %s_ts2" % ident)
  74. @configure_follower.for_db("oracle")
  75. def _oracle_configure_follower(config, ident):
  76. config.test_schema = "%s_ts1" % ident
  77. config.test_schema_2 = "%s_ts2" % ident
  78. def _ora_drop_ignore(conn, dbname):
  79. try:
  80. conn.exec_driver_sql("drop user %s cascade" % dbname)
  81. log.info("Reaped db: %s", dbname)
  82. return True
  83. except exc.DatabaseError as err:
  84. log.warning("couldn't drop db: %s", err)
  85. return False
  86. @drop_all_schema_objects_pre_tables.for_db("oracle")
  87. def _ora_drop_all_schema_objects_pre_tables(cfg, eng):
  88. _purge_recyclebin(eng)
  89. _purge_recyclebin(eng, cfg.test_schema)
  90. @drop_all_schema_objects_post_tables.for_db("oracle")
  91. def _ora_drop_all_schema_objects_post_tables(cfg, eng):
  92. with eng.begin() as conn:
  93. for syn in conn.dialect._get_synonyms(conn, None, None, None):
  94. conn.exec_driver_sql(f"drop synonym {syn['synonym_name']}")
  95. for syn in conn.dialect._get_synonyms(
  96. conn, cfg.test_schema, None, None
  97. ):
  98. conn.exec_driver_sql(
  99. f"drop synonym {cfg.test_schema}.{syn['synonym_name']}"
  100. )
  101. for tmp_table in inspect(conn).get_temp_table_names():
  102. conn.exec_driver_sql(f"drop table {tmp_table}")
  103. @drop_db.for_db("oracle")
  104. def _oracle_drop_db(cfg, eng, ident):
  105. with eng.begin() as conn:
  106. # cx_Oracle seems to occasionally leak open connections when a large
  107. # suite it run, even if we confirm we have zero references to
  108. # connection objects.
  109. # while there is a "kill session" command in Oracle Database,
  110. # it unfortunately does not release the connection sufficiently.
  111. _ora_drop_ignore(conn, ident)
  112. _ora_drop_ignore(conn, "%s_ts1" % ident)
  113. _ora_drop_ignore(conn, "%s_ts2" % ident)
  114. @stop_test_class_outside_fixtures.for_db("oracle")
  115. def _ora_stop_test_class_outside_fixtures(config, db, cls):
  116. try:
  117. _purge_recyclebin(db)
  118. except exc.DatabaseError as err:
  119. log.warning("purge recyclebin command failed: %s", err)
  120. def _purge_recyclebin(eng, schema=None):
  121. with eng.begin() as conn:
  122. if schema is None:
  123. # run magic command to get rid of identity sequences
  124. # https://floo.bar/2019/11/29/drop-the-underlying-sequence-of-an-identity-column/ # noqa: E501
  125. conn.exec_driver_sql("purge recyclebin")
  126. else:
  127. # per user: https://community.oracle.com/tech/developers/discussion/2255402/how-to-clear-dba-recyclebin-for-a-particular-user # noqa: E501
  128. for owner, object_name, type_ in conn.exec_driver_sql(
  129. "select owner, object_name,type from "
  130. "dba_recyclebin where owner=:schema and type='TABLE'",
  131. {"schema": conn.dialect.denormalize_name(schema)},
  132. ).all():
  133. conn.exec_driver_sql(f'purge {type_} {owner}."{object_name}"')
  134. @is_preferred_driver.for_db("oracle")
  135. def _oracle_is_preferred_driver(cfg, engine):
  136. """establish oracledb as the preferred driver to use for tests, even
  137. though cx_Oracle is still the "default" driver"""
  138. return engine.dialect.driver == "oracledb" and not engine.dialect.is_async
  139. def _connect_with_retry(dialect, conn_rec, cargs, cparams):
  140. assert dialect.driver == "cx_oracle"
  141. def _is_couldnt_connect(err):
  142. return "DPY-6005" in str(err) or "ORA-12516" in str(err)
  143. err_ = None
  144. for _ in range(5):
  145. try:
  146. return dialect.loaded_dbapi.connect(*cargs, **cparams)
  147. except (
  148. dialect.loaded_dbapi.DatabaseError,
  149. dialect.loaded_dbapi.OperationalError,
  150. ) as err:
  151. err_ = err
  152. if _is_couldnt_connect(err):
  153. warn_test_suite("Oracle database reconnecting...")
  154. time.sleep(2)
  155. continue
  156. else:
  157. raise
  158. if err_ is not None:
  159. raise Exception("connect failed after five attempts") from err_
  160. @post_configure_testing_engine.for_db("oracle")
  161. def _oracle_post_configure_testing_engine(url, engine, options, scope):
  162. from ... import event
  163. if engine.dialect.driver == "cx_oracle":
  164. event.listen(engine, "do_connect", _connect_with_retry)
  165. @post_configure_engine.for_db("oracle")
  166. def _oracle_post_configure_engine(url, engine, follower_ident):
  167. from ... import event
  168. @event.listens_for(engine, "checkin")
  169. def checkin(dbapi_connection, connection_record):
  170. # this was meant to work around this issue:
  171. # https://github.com/oracle/python-cx_Oracle/issues/530
  172. # invalidate oracle connections that had 2pc set up
  173. # however things are too complex with some of the 2pc tests,
  174. # so just block cx_oracle from being used in 2pc tests (use oracledb
  175. # instead)
  176. # if "cx_oracle_xid" in connection_record.info:
  177. # connection_record.invalidate()
  178. # clear statement cache on all connections that were used
  179. # https://github.com/oracle/python-cx_Oracle/issues/519
  180. # TODO: oracledb claims to have this feature built in somehow,
  181. # see if that's in use and/or if it needs to be enabled
  182. # (or if this doesn't even apply to the newer oracle's we're using)
  183. try:
  184. sc = dbapi_connection.stmtcachesize
  185. except:
  186. # connection closed
  187. pass
  188. else:
  189. dbapi_connection.stmtcachesize = 0
  190. dbapi_connection.stmtcachesize = sc
  191. @run_reap_dbs.for_db("oracle")
  192. def _reap_oracle_dbs(url, idents):
  193. log.info("db reaper connecting to %r", url)
  194. eng = create_engine(url)
  195. with eng.begin() as conn:
  196. log.info("identifiers in file: %s", ", ".join(idents))
  197. to_reap = conn.exec_driver_sql(
  198. "select u.username from all_users u where username "
  199. "like 'TEST_%' and not exists (select username "
  200. "from v$session where username=u.username)"
  201. )
  202. all_names = {username.lower() for (username,) in to_reap}
  203. to_drop = set()
  204. for name in all_names:
  205. if name.endswith("_ts1") or name.endswith("_ts2"):
  206. continue
  207. elif name in idents:
  208. to_drop.add(name)
  209. if "%s_ts1" % name in all_names:
  210. to_drop.add("%s_ts1" % name)
  211. if "%s_ts2" % name in all_names:
  212. to_drop.add("%s_ts2" % name)
  213. dropped = total = 0
  214. for total, username in enumerate(to_drop, 1):
  215. if _ora_drop_ignore(conn, username):
  216. dropped += 1
  217. log.info(
  218. "Dropped %d out of %d stale databases detected", dropped, total
  219. )
  220. @follower_url_from_main.for_db("oracle")
  221. def _oracle_follower_url_from_main(url, ident):
  222. url = sa_url.make_url(url)
  223. return url.set(username=ident, password="xe")
  224. @temp_table_keyword_args.for_db("oracle")
  225. def _oracle_temp_table_keyword_args(cfg, eng):
  226. return {
  227. "prefixes": ["GLOBAL TEMPORARY"],
  228. "oracle_on_commit": "PRESERVE ROWS",
  229. }
  230. @set_default_schema_on_connection.for_db("oracle")
  231. def _oracle_set_default_schema_on_connection(
  232. cfg, dbapi_connection, schema_name
  233. ):
  234. cursor = dbapi_connection.cursor()
  235. cursor.execute("ALTER SESSION SET CURRENT_SCHEMA=%s" % schema_name)
  236. cursor.close()
  237. @update_db_opts.for_db("oracle")
  238. def _update_db_opts(db_url, db_opts, options):
  239. """Set database options (db_opts) for a test database that we created."""
  240. if (
  241. options.oracledb_thick_mode
  242. and sa_url.make_url(db_url).get_driver_name() == "oracledb"
  243. ):
  244. db_opts["thick_mode"] = True