text.js 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244
  1. /**
  2. * @import {
  3. * Code,
  4. * InitialConstruct,
  5. * Initializer,
  6. * Resolver,
  7. * State,
  8. * TokenizeContext
  9. * } from 'micromark-util-types'
  10. */
  11. import {ok as assert} from 'devlop'
  12. import {codes, constants, types} from 'micromark-util-symbol'
  13. export const resolver = {resolveAll: createResolver()}
  14. export const string = initializeFactory('string')
  15. export const text = initializeFactory('text')
  16. /**
  17. * @param {'string' | 'text'} field
  18. * Field.
  19. * @returns {InitialConstruct}
  20. * Construct.
  21. */
  22. function initializeFactory(field) {
  23. return {
  24. resolveAll: createResolver(
  25. field === 'text' ? resolveAllLineSuffixes : undefined
  26. ),
  27. tokenize: initializeText
  28. }
  29. /**
  30. * @this {TokenizeContext}
  31. * Context.
  32. * @type {Initializer}
  33. */
  34. function initializeText(effects) {
  35. const self = this
  36. const constructs = this.parser.constructs[field]
  37. const text = effects.attempt(constructs, start, notText)
  38. return start
  39. /** @type {State} */
  40. function start(code) {
  41. return atBreak(code) ? text(code) : notText(code)
  42. }
  43. /** @type {State} */
  44. function notText(code) {
  45. if (code === codes.eof) {
  46. effects.consume(code)
  47. return
  48. }
  49. effects.enter(types.data)
  50. effects.consume(code)
  51. return data
  52. }
  53. /** @type {State} */
  54. function data(code) {
  55. if (atBreak(code)) {
  56. effects.exit(types.data)
  57. return text(code)
  58. }
  59. // Data.
  60. effects.consume(code)
  61. return data
  62. }
  63. /**
  64. * @param {Code} code
  65. * Code.
  66. * @returns {boolean}
  67. * Whether the code is a break.
  68. */
  69. function atBreak(code) {
  70. if (code === codes.eof) {
  71. return true
  72. }
  73. const list = constructs[code]
  74. let index = -1
  75. if (list) {
  76. // Always populated by defaults.
  77. assert(Array.isArray(list), 'expected `disable.null` to be populated')
  78. while (++index < list.length) {
  79. const item = list[index]
  80. if (!item.previous || item.previous.call(self, self.previous)) {
  81. return true
  82. }
  83. }
  84. }
  85. return false
  86. }
  87. }
  88. }
  89. /**
  90. * @param {Resolver | undefined} [extraResolver]
  91. * Resolver.
  92. * @returns {Resolver}
  93. * Resolver.
  94. */
  95. function createResolver(extraResolver) {
  96. return resolveAllText
  97. /** @type {Resolver} */
  98. function resolveAllText(events, context) {
  99. let index = -1
  100. /** @type {number | undefined} */
  101. let enter
  102. // A rather boring computation (to merge adjacent `data` events) which
  103. // improves mm performance by 29%.
  104. while (++index <= events.length) {
  105. if (enter === undefined) {
  106. if (events[index] && events[index][1].type === types.data) {
  107. enter = index
  108. index++
  109. }
  110. } else if (!events[index] || events[index][1].type !== types.data) {
  111. // Don’t do anything if there is one data token.
  112. if (index !== enter + 2) {
  113. events[enter][1].end = events[index - 1][1].end
  114. events.splice(enter + 2, index - enter - 2)
  115. index = enter + 2
  116. }
  117. enter = undefined
  118. }
  119. }
  120. return extraResolver ? extraResolver(events, context) : events
  121. }
  122. }
  123. /**
  124. * A rather ugly set of instructions which again looks at chunks in the input
  125. * stream.
  126. * The reason to do this here is that it is *much* faster to parse in reverse.
  127. * And that we can’t hook into `null` to split the line suffix before an EOF.
  128. * To do: figure out if we can make this into a clean utility, or even in core.
  129. * As it will be useful for GFMs literal autolink extension (and maybe even
  130. * tables?)
  131. *
  132. * @type {Resolver}
  133. */
  134. function resolveAllLineSuffixes(events, context) {
  135. let eventIndex = 0 // Skip first.
  136. while (++eventIndex <= events.length) {
  137. if (
  138. (eventIndex === events.length ||
  139. events[eventIndex][1].type === types.lineEnding) &&
  140. events[eventIndex - 1][1].type === types.data
  141. ) {
  142. const data = events[eventIndex - 1][1]
  143. const chunks = context.sliceStream(data)
  144. let index = chunks.length
  145. let bufferIndex = -1
  146. let size = 0
  147. /** @type {boolean | undefined} */
  148. let tabs
  149. while (index--) {
  150. const chunk = chunks[index]
  151. if (typeof chunk === 'string') {
  152. bufferIndex = chunk.length
  153. while (chunk.charCodeAt(bufferIndex - 1) === codes.space) {
  154. size++
  155. bufferIndex--
  156. }
  157. if (bufferIndex) break
  158. bufferIndex = -1
  159. }
  160. // Number
  161. else if (chunk === codes.horizontalTab) {
  162. tabs = true
  163. size++
  164. } else if (chunk === codes.virtualSpace) {
  165. // Empty
  166. } else {
  167. // Replacement character, exit.
  168. index++
  169. break
  170. }
  171. }
  172. // Allow final trailing whitespace.
  173. if (context._contentTypeTextTrailing && eventIndex === events.length) {
  174. size = 0
  175. }
  176. if (size) {
  177. const token = {
  178. type:
  179. eventIndex === events.length ||
  180. tabs ||
  181. size < constants.hardBreakPrefixSizeMin
  182. ? types.lineSuffix
  183. : types.hardBreakTrailing,
  184. start: {
  185. _bufferIndex: index
  186. ? bufferIndex
  187. : data.start._bufferIndex + bufferIndex,
  188. _index: data.start._index + index,
  189. line: data.end.line,
  190. column: data.end.column - size,
  191. offset: data.end.offset - size
  192. },
  193. end: {...data.end}
  194. }
  195. data.end = {...token.start}
  196. if (data.start.offset === data.end.offset) {
  197. Object.assign(data, token)
  198. } else {
  199. events.splice(
  200. eventIndex,
  201. 0,
  202. ['enter', token, context],
  203. ['exit', token, context]
  204. )
  205. eventIndex += 2
  206. }
  207. }
  208. eventIndex++
  209. }
  210. }
  211. return events
  212. }