typing.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608
  1. import sys
  2. import typing
  3. from collections.abc import Callable
  4. from os import PathLike
  5. from typing import ( # type: ignore
  6. TYPE_CHECKING,
  7. AbstractSet,
  8. Any,
  9. Callable as TypingCallable,
  10. ClassVar,
  11. Dict,
  12. ForwardRef,
  13. Generator,
  14. Iterable,
  15. List,
  16. Mapping,
  17. NewType,
  18. Optional,
  19. Sequence,
  20. Set,
  21. Tuple,
  22. Type,
  23. TypeVar,
  24. Union,
  25. _eval_type,
  26. cast,
  27. get_type_hints,
  28. )
  29. from typing_extensions import (
  30. Annotated,
  31. Final,
  32. Literal,
  33. NotRequired as TypedDictNotRequired,
  34. Required as TypedDictRequired,
  35. )
  36. try:
  37. from typing import _TypingBase as typing_base # type: ignore
  38. except ImportError:
  39. from typing import _Final as typing_base # type: ignore
  40. try:
  41. from typing import GenericAlias as TypingGenericAlias # type: ignore
  42. except ImportError:
  43. # python < 3.9 does not have GenericAlias (list[int], tuple[str, ...] and so on)
  44. TypingGenericAlias = ()
  45. try:
  46. from types import UnionType as TypesUnionType # type: ignore
  47. except ImportError:
  48. # python < 3.10 does not have UnionType (str | int, byte | bool and so on)
  49. TypesUnionType = ()
  50. if sys.version_info < (3, 9):
  51. def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any:
  52. return type_._evaluate(globalns, localns)
  53. else:
  54. def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any:
  55. # Even though it is the right signature for python 3.9, mypy complains with
  56. # `error: Too many arguments for "_evaluate" of "ForwardRef"` hence the cast...
  57. # Python 3.13/3.12.4+ made `recursive_guard` a kwarg, so name it explicitly to avoid:
  58. # TypeError: ForwardRef._evaluate() missing 1 required keyword-only argument: 'recursive_guard'
  59. return cast(Any, type_)._evaluate(globalns, localns, recursive_guard=set())
  60. if sys.version_info < (3, 9):
  61. # Ensure we always get all the whole `Annotated` hint, not just the annotated type.
  62. # For 3.7 to 3.8, `get_type_hints` doesn't recognize `typing_extensions.Annotated`,
  63. # so it already returns the full annotation
  64. get_all_type_hints = get_type_hints
  65. else:
  66. def get_all_type_hints(obj: Any, globalns: Any = None, localns: Any = None) -> Any:
  67. return get_type_hints(obj, globalns, localns, include_extras=True)
  68. _T = TypeVar('_T')
  69. AnyCallable = TypingCallable[..., Any]
  70. NoArgAnyCallable = TypingCallable[[], Any]
  71. # workaround for https://github.com/python/mypy/issues/9496
  72. AnyArgTCallable = TypingCallable[..., _T]
  73. # Annotated[...] is implemented by returning an instance of one of these classes, depending on
  74. # python/typing_extensions version.
  75. AnnotatedTypeNames = {'AnnotatedMeta', '_AnnotatedAlias'}
  76. LITERAL_TYPES: Set[Any] = {Literal}
  77. if hasattr(typing, 'Literal'):
  78. LITERAL_TYPES.add(typing.Literal)
  79. if sys.version_info < (3, 8):
  80. def get_origin(t: Type[Any]) -> Optional[Type[Any]]:
  81. if type(t).__name__ in AnnotatedTypeNames:
  82. # weirdly this is a runtime requirement, as well as for mypy
  83. return cast(Type[Any], Annotated)
  84. return getattr(t, '__origin__', None)
  85. else:
  86. from typing import get_origin as _typing_get_origin
  87. def get_origin(tp: Type[Any]) -> Optional[Type[Any]]:
  88. """
  89. We can't directly use `typing.get_origin` since we need a fallback to support
  90. custom generic classes like `ConstrainedList`
  91. It should be useless once https://github.com/cython/cython/issues/3537 is
  92. solved and https://github.com/pydantic/pydantic/pull/1753 is merged.
  93. """
  94. if type(tp).__name__ in AnnotatedTypeNames:
  95. return cast(Type[Any], Annotated) # mypy complains about _SpecialForm
  96. return _typing_get_origin(tp) or getattr(tp, '__origin__', None)
  97. if sys.version_info < (3, 8):
  98. from typing import _GenericAlias
  99. def get_args(t: Type[Any]) -> Tuple[Any, ...]:
  100. """Compatibility version of get_args for python 3.7.
  101. Mostly compatible with the python 3.8 `typing` module version
  102. and able to handle almost all use cases.
  103. """
  104. if type(t).__name__ in AnnotatedTypeNames:
  105. return t.__args__ + t.__metadata__
  106. if isinstance(t, _GenericAlias):
  107. res = t.__args__
  108. if t.__origin__ is Callable and res and res[0] is not Ellipsis:
  109. res = (list(res[:-1]), res[-1])
  110. return res
  111. return getattr(t, '__args__', ())
  112. else:
  113. from typing import get_args as _typing_get_args
  114. def _generic_get_args(tp: Type[Any]) -> Tuple[Any, ...]:
  115. """
  116. In python 3.9, `typing.Dict`, `typing.List`, ...
  117. do have an empty `__args__` by default (instead of the generic ~T for example).
  118. In order to still support `Dict` for example and consider it as `Dict[Any, Any]`,
  119. we retrieve the `_nparams` value that tells us how many parameters it needs.
  120. """
  121. if hasattr(tp, '_nparams'):
  122. return (Any,) * tp._nparams
  123. # Special case for `tuple[()]`, which used to return ((),) with `typing.Tuple`
  124. # in python 3.10- but now returns () for `tuple` and `Tuple`.
  125. # This will probably be clarified in pydantic v2
  126. try:
  127. if tp == Tuple[()] or sys.version_info >= (3, 9) and tp == tuple[()]: # type: ignore[misc]
  128. return ((),)
  129. # there is a TypeError when compiled with cython
  130. except TypeError: # pragma: no cover
  131. pass
  132. return ()
  133. def get_args(tp: Type[Any]) -> Tuple[Any, ...]:
  134. """Get type arguments with all substitutions performed.
  135. For unions, basic simplifications used by Union constructor are performed.
  136. Examples::
  137. get_args(Dict[str, int]) == (str, int)
  138. get_args(int) == ()
  139. get_args(Union[int, Union[T, int], str][int]) == (int, str)
  140. get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int])
  141. get_args(Callable[[], T][int]) == ([], int)
  142. """
  143. if type(tp).__name__ in AnnotatedTypeNames:
  144. return tp.__args__ + tp.__metadata__
  145. # the fallback is needed for the same reasons as `get_origin` (see above)
  146. return _typing_get_args(tp) or getattr(tp, '__args__', ()) or _generic_get_args(tp)
  147. if sys.version_info < (3, 9):
  148. def convert_generics(tp: Type[Any]) -> Type[Any]:
  149. """Python 3.9 and older only supports generics from `typing` module.
  150. They convert strings to ForwardRef automatically.
  151. Examples::
  152. typing.List['Hero'] == typing.List[ForwardRef('Hero')]
  153. """
  154. return tp
  155. else:
  156. from typing import _UnionGenericAlias # type: ignore
  157. from typing_extensions import _AnnotatedAlias
  158. def convert_generics(tp: Type[Any]) -> Type[Any]:
  159. """
  160. Recursively searches for `str` type hints and replaces them with ForwardRef.
  161. Examples::
  162. convert_generics(list['Hero']) == list[ForwardRef('Hero')]
  163. convert_generics(dict['Hero', 'Team']) == dict[ForwardRef('Hero'), ForwardRef('Team')]
  164. convert_generics(typing.Dict['Hero', 'Team']) == typing.Dict[ForwardRef('Hero'), ForwardRef('Team')]
  165. convert_generics(list[str | 'Hero'] | int) == list[str | ForwardRef('Hero')] | int
  166. """
  167. origin = get_origin(tp)
  168. if not origin or not hasattr(tp, '__args__'):
  169. return tp
  170. args = get_args(tp)
  171. # typing.Annotated needs special treatment
  172. if origin is Annotated:
  173. return _AnnotatedAlias(convert_generics(args[0]), args[1:])
  174. # recursively replace `str` instances inside of `GenericAlias` with `ForwardRef(arg)`
  175. converted = tuple(
  176. ForwardRef(arg) if isinstance(arg, str) and isinstance(tp, TypingGenericAlias) else convert_generics(arg)
  177. for arg in args
  178. )
  179. if converted == args:
  180. return tp
  181. elif isinstance(tp, TypingGenericAlias):
  182. return TypingGenericAlias(origin, converted)
  183. elif isinstance(tp, TypesUnionType):
  184. # recreate types.UnionType (PEP604, Python >= 3.10)
  185. return _UnionGenericAlias(origin, converted)
  186. else:
  187. try:
  188. setattr(tp, '__args__', converted)
  189. except AttributeError:
  190. pass
  191. return tp
  192. if sys.version_info < (3, 10):
  193. def is_union(tp: Optional[Type[Any]]) -> bool:
  194. return tp is Union
  195. WithArgsTypes = (TypingGenericAlias,)
  196. else:
  197. import types
  198. import typing
  199. def is_union(tp: Optional[Type[Any]]) -> bool:
  200. return tp is Union or tp is types.UnionType # noqa: E721
  201. WithArgsTypes = (typing._GenericAlias, types.GenericAlias, types.UnionType)
  202. StrPath = Union[str, PathLike]
  203. if TYPE_CHECKING:
  204. from pydantic.v1.fields import ModelField
  205. TupleGenerator = Generator[Tuple[str, Any], None, None]
  206. DictStrAny = Dict[str, Any]
  207. DictAny = Dict[Any, Any]
  208. SetStr = Set[str]
  209. ListStr = List[str]
  210. IntStr = Union[int, str]
  211. AbstractSetIntStr = AbstractSet[IntStr]
  212. DictIntStrAny = Dict[IntStr, Any]
  213. MappingIntStrAny = Mapping[IntStr, Any]
  214. CallableGenerator = Generator[AnyCallable, None, None]
  215. ReprArgs = Sequence[Tuple[Optional[str], Any]]
  216. MYPY = False
  217. if MYPY:
  218. AnyClassMethod = classmethod[Any]
  219. else:
  220. # classmethod[TargetType, CallableParamSpecType, CallableReturnType]
  221. AnyClassMethod = classmethod[Any, Any, Any]
  222. __all__ = (
  223. 'AnyCallable',
  224. 'NoArgAnyCallable',
  225. 'NoneType',
  226. 'is_none_type',
  227. 'display_as_type',
  228. 'resolve_annotations',
  229. 'is_callable_type',
  230. 'is_literal_type',
  231. 'all_literal_values',
  232. 'is_namedtuple',
  233. 'is_typeddict',
  234. 'is_typeddict_special',
  235. 'is_new_type',
  236. 'new_type_supertype',
  237. 'is_classvar',
  238. 'is_finalvar',
  239. 'update_field_forward_refs',
  240. 'update_model_forward_refs',
  241. 'TupleGenerator',
  242. 'DictStrAny',
  243. 'DictAny',
  244. 'SetStr',
  245. 'ListStr',
  246. 'IntStr',
  247. 'AbstractSetIntStr',
  248. 'DictIntStrAny',
  249. 'CallableGenerator',
  250. 'ReprArgs',
  251. 'AnyClassMethod',
  252. 'CallableGenerator',
  253. 'WithArgsTypes',
  254. 'get_args',
  255. 'get_origin',
  256. 'get_sub_types',
  257. 'typing_base',
  258. 'get_all_type_hints',
  259. 'is_union',
  260. 'StrPath',
  261. 'MappingIntStrAny',
  262. )
  263. NoneType = None.__class__
  264. NONE_TYPES: Tuple[Any, Any, Any] = (None, NoneType, Literal[None])
  265. if sys.version_info < (3, 8):
  266. # Even though this implementation is slower, we need it for python 3.7:
  267. # In python 3.7 "Literal" is not a builtin type and uses a different
  268. # mechanism.
  269. # for this reason `Literal[None] is Literal[None]` evaluates to `False`,
  270. # breaking the faster implementation used for the other python versions.
  271. def is_none_type(type_: Any) -> bool:
  272. return type_ in NONE_TYPES
  273. elif sys.version_info[:2] == (3, 8):
  274. def is_none_type(type_: Any) -> bool:
  275. for none_type in NONE_TYPES:
  276. if type_ is none_type:
  277. return True
  278. # With python 3.8, specifically 3.8.10, Literal "is" check sare very flakey
  279. # can change on very subtle changes like use of types in other modules,
  280. # hopefully this check avoids that issue.
  281. if is_literal_type(type_): # pragma: no cover
  282. return all_literal_values(type_) == (None,)
  283. return False
  284. else:
  285. def is_none_type(type_: Any) -> bool:
  286. return type_ in NONE_TYPES
  287. def display_as_type(v: Type[Any]) -> str:
  288. if not isinstance(v, typing_base) and not isinstance(v, WithArgsTypes) and not isinstance(v, type):
  289. v = v.__class__
  290. if is_union(get_origin(v)):
  291. return f'Union[{", ".join(map(display_as_type, get_args(v)))}]'
  292. if isinstance(v, WithArgsTypes):
  293. # Generic alias are constructs like `list[int]`
  294. return str(v).replace('typing.', '')
  295. try:
  296. return v.__name__
  297. except AttributeError:
  298. # happens with typing objects
  299. return str(v).replace('typing.', '')
  300. def resolve_annotations(raw_annotations: Dict[str, Type[Any]], module_name: Optional[str]) -> Dict[str, Type[Any]]:
  301. """
  302. Partially taken from typing.get_type_hints.
  303. Resolve string or ForwardRef annotations into type objects if possible.
  304. """
  305. base_globals: Optional[Dict[str, Any]] = None
  306. if module_name:
  307. try:
  308. module = sys.modules[module_name]
  309. except KeyError:
  310. # happens occasionally, see https://github.com/pydantic/pydantic/issues/2363
  311. pass
  312. else:
  313. base_globals = module.__dict__
  314. annotations = {}
  315. for name, value in raw_annotations.items():
  316. if isinstance(value, str):
  317. if (3, 10) > sys.version_info >= (3, 9, 8) or sys.version_info >= (3, 10, 1):
  318. value = ForwardRef(value, is_argument=False, is_class=True)
  319. else:
  320. value = ForwardRef(value, is_argument=False)
  321. try:
  322. if sys.version_info >= (3, 13):
  323. value = _eval_type(value, base_globals, None, type_params=())
  324. else:
  325. value = _eval_type(value, base_globals, None)
  326. except NameError:
  327. # this is ok, it can be fixed with update_forward_refs
  328. pass
  329. annotations[name] = value
  330. return annotations
  331. def is_callable_type(type_: Type[Any]) -> bool:
  332. return type_ is Callable or get_origin(type_) is Callable
  333. def is_literal_type(type_: Type[Any]) -> bool:
  334. return Literal is not None and get_origin(type_) in LITERAL_TYPES
  335. def literal_values(type_: Type[Any]) -> Tuple[Any, ...]:
  336. return get_args(type_)
  337. def all_literal_values(type_: Type[Any]) -> Tuple[Any, ...]:
  338. """
  339. This method is used to retrieve all Literal values as
  340. Literal can be used recursively (see https://www.python.org/dev/peps/pep-0586)
  341. e.g. `Literal[Literal[Literal[1, 2, 3], "foo"], 5, None]`
  342. """
  343. if not is_literal_type(type_):
  344. return (type_,)
  345. values = literal_values(type_)
  346. return tuple(x for value in values for x in all_literal_values(value))
  347. def is_namedtuple(type_: Type[Any]) -> bool:
  348. """
  349. Check if a given class is a named tuple.
  350. It can be either a `typing.NamedTuple` or `collections.namedtuple`
  351. """
  352. from pydantic.v1.utils import lenient_issubclass
  353. return lenient_issubclass(type_, tuple) and hasattr(type_, '_fields')
  354. def is_typeddict(type_: Type[Any]) -> bool:
  355. """
  356. Check if a given class is a typed dict (from `typing` or `typing_extensions`)
  357. In 3.10, there will be a public method (https://docs.python.org/3.10/library/typing.html#typing.is_typeddict)
  358. """
  359. from pydantic.v1.utils import lenient_issubclass
  360. return lenient_issubclass(type_, dict) and hasattr(type_, '__total__')
  361. def _check_typeddict_special(type_: Any) -> bool:
  362. return type_ is TypedDictRequired or type_ is TypedDictNotRequired
  363. def is_typeddict_special(type_: Any) -> bool:
  364. """
  365. Check if type is a TypedDict special form (Required or NotRequired).
  366. """
  367. return _check_typeddict_special(type_) or _check_typeddict_special(get_origin(type_))
  368. test_type = NewType('test_type', str)
  369. def is_new_type(type_: Type[Any]) -> bool:
  370. """
  371. Check whether type_ was created using typing.NewType
  372. """
  373. return isinstance(type_, test_type.__class__) and hasattr(type_, '__supertype__') # type: ignore
  374. def new_type_supertype(type_: Type[Any]) -> Type[Any]:
  375. while hasattr(type_, '__supertype__'):
  376. type_ = type_.__supertype__
  377. return type_
  378. def _check_classvar(v: Optional[Type[Any]]) -> bool:
  379. if v is None:
  380. return False
  381. return v.__class__ == ClassVar.__class__ and getattr(v, '_name', None) == 'ClassVar'
  382. def _check_finalvar(v: Optional[Type[Any]]) -> bool:
  383. """
  384. Check if a given type is a `typing.Final` type.
  385. """
  386. if v is None:
  387. return False
  388. return v.__class__ == Final.__class__ and (sys.version_info < (3, 8) or getattr(v, '_name', None) == 'Final')
  389. def is_classvar(ann_type: Type[Any]) -> bool:
  390. if _check_classvar(ann_type) or _check_classvar(get_origin(ann_type)):
  391. return True
  392. # this is an ugly workaround for class vars that contain forward references and are therefore themselves
  393. # forward references, see #3679
  394. if ann_type.__class__ == ForwardRef and ann_type.__forward_arg__.startswith('ClassVar['):
  395. return True
  396. return False
  397. def is_finalvar(ann_type: Type[Any]) -> bool:
  398. return _check_finalvar(ann_type) or _check_finalvar(get_origin(ann_type))
  399. def update_field_forward_refs(field: 'ModelField', globalns: Any, localns: Any) -> None:
  400. """
  401. Try to update ForwardRefs on fields based on this ModelField, globalns and localns.
  402. """
  403. prepare = False
  404. if field.type_.__class__ == ForwardRef:
  405. prepare = True
  406. field.type_ = evaluate_forwardref(field.type_, globalns, localns or None)
  407. if field.outer_type_.__class__ == ForwardRef:
  408. prepare = True
  409. field.outer_type_ = evaluate_forwardref(field.outer_type_, globalns, localns or None)
  410. if prepare:
  411. field.prepare()
  412. if field.sub_fields:
  413. for sub_f in field.sub_fields:
  414. update_field_forward_refs(sub_f, globalns=globalns, localns=localns)
  415. if field.discriminator_key is not None:
  416. field.prepare_discriminated_union_sub_fields()
  417. def update_model_forward_refs(
  418. model: Type[Any],
  419. fields: Iterable['ModelField'],
  420. json_encoders: Dict[Union[Type[Any], str, ForwardRef], AnyCallable],
  421. localns: 'DictStrAny',
  422. exc_to_suppress: Tuple[Type[BaseException], ...] = (),
  423. ) -> None:
  424. """
  425. Try to update model fields ForwardRefs based on model and localns.
  426. """
  427. if model.__module__ in sys.modules:
  428. globalns = sys.modules[model.__module__].__dict__.copy()
  429. else:
  430. globalns = {}
  431. globalns.setdefault(model.__name__, model)
  432. for f in fields:
  433. try:
  434. update_field_forward_refs(f, globalns=globalns, localns=localns)
  435. except exc_to_suppress:
  436. pass
  437. for key in set(json_encoders.keys()):
  438. if isinstance(key, str):
  439. fr: ForwardRef = ForwardRef(key)
  440. elif isinstance(key, ForwardRef):
  441. fr = key
  442. else:
  443. continue
  444. try:
  445. new_key = evaluate_forwardref(fr, globalns, localns or None)
  446. except exc_to_suppress: # pragma: no cover
  447. continue
  448. json_encoders[new_key] = json_encoders.pop(key)
  449. def get_class(type_: Type[Any]) -> Union[None, bool, Type[Any]]:
  450. """
  451. Tries to get the class of a Type[T] annotation. Returns True if Type is used
  452. without brackets. Otherwise returns None.
  453. """
  454. if type_ is type:
  455. return True
  456. if get_origin(type_) is None:
  457. return None
  458. args = get_args(type_)
  459. if not args or not isinstance(args[0], type):
  460. return True
  461. else:
  462. return args[0]
  463. def get_sub_types(tp: Any) -> List[Any]:
  464. """
  465. Return all the types that are allowed by type `tp`
  466. `tp` can be a `Union` of allowed types or an `Annotated` type
  467. """
  468. origin = get_origin(tp)
  469. if origin is Annotated:
  470. return get_sub_types(get_args(tp)[0])
  471. elif is_union(origin):
  472. return [x for t in get_args(tp) for x in get_sub_types(t)]
  473. else:
  474. return [tp]