text.js 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212
  1. /**
  2. * @import {
  3. * Code,
  4. * InitialConstruct,
  5. * Initializer,
  6. * Resolver,
  7. * State,
  8. * TokenizeContext
  9. * } from 'micromark-util-types'
  10. */
  11. export const resolver = {
  12. resolveAll: createResolver()
  13. };
  14. export const string = initializeFactory('string');
  15. export const text = initializeFactory('text');
  16. /**
  17. * @param {'string' | 'text'} field
  18. * Field.
  19. * @returns {InitialConstruct}
  20. * Construct.
  21. */
  22. function initializeFactory(field) {
  23. return {
  24. resolveAll: createResolver(field === 'text' ? resolveAllLineSuffixes : undefined),
  25. tokenize: initializeText
  26. };
  27. /**
  28. * @this {TokenizeContext}
  29. * Context.
  30. * @type {Initializer}
  31. */
  32. function initializeText(effects) {
  33. const self = this;
  34. const constructs = this.parser.constructs[field];
  35. const text = effects.attempt(constructs, start, notText);
  36. return start;
  37. /** @type {State} */
  38. function start(code) {
  39. return atBreak(code) ? text(code) : notText(code);
  40. }
  41. /** @type {State} */
  42. function notText(code) {
  43. if (code === null) {
  44. effects.consume(code);
  45. return;
  46. }
  47. effects.enter("data");
  48. effects.consume(code);
  49. return data;
  50. }
  51. /** @type {State} */
  52. function data(code) {
  53. if (atBreak(code)) {
  54. effects.exit("data");
  55. return text(code);
  56. }
  57. // Data.
  58. effects.consume(code);
  59. return data;
  60. }
  61. /**
  62. * @param {Code} code
  63. * Code.
  64. * @returns {boolean}
  65. * Whether the code is a break.
  66. */
  67. function atBreak(code) {
  68. if (code === null) {
  69. return true;
  70. }
  71. const list = constructs[code];
  72. let index = -1;
  73. if (list) {
  74. // Always populated by defaults.
  75. while (++index < list.length) {
  76. const item = list[index];
  77. if (!item.previous || item.previous.call(self, self.previous)) {
  78. return true;
  79. }
  80. }
  81. }
  82. return false;
  83. }
  84. }
  85. }
  86. /**
  87. * @param {Resolver | undefined} [extraResolver]
  88. * Resolver.
  89. * @returns {Resolver}
  90. * Resolver.
  91. */
  92. function createResolver(extraResolver) {
  93. return resolveAllText;
  94. /** @type {Resolver} */
  95. function resolveAllText(events, context) {
  96. let index = -1;
  97. /** @type {number | undefined} */
  98. let enter;
  99. // A rather boring computation (to merge adjacent `data` events) which
  100. // improves mm performance by 29%.
  101. while (++index <= events.length) {
  102. if (enter === undefined) {
  103. if (events[index] && events[index][1].type === "data") {
  104. enter = index;
  105. index++;
  106. }
  107. } else if (!events[index] || events[index][1].type !== "data") {
  108. // Don’t do anything if there is one data token.
  109. if (index !== enter + 2) {
  110. events[enter][1].end = events[index - 1][1].end;
  111. events.splice(enter + 2, index - enter - 2);
  112. index = enter + 2;
  113. }
  114. enter = undefined;
  115. }
  116. }
  117. return extraResolver ? extraResolver(events, context) : events;
  118. }
  119. }
  120. /**
  121. * A rather ugly set of instructions which again looks at chunks in the input
  122. * stream.
  123. * The reason to do this here is that it is *much* faster to parse in reverse.
  124. * And that we can’t hook into `null` to split the line suffix before an EOF.
  125. * To do: figure out if we can make this into a clean utility, or even in core.
  126. * As it will be useful for GFMs literal autolink extension (and maybe even
  127. * tables?)
  128. *
  129. * @type {Resolver}
  130. */
  131. function resolveAllLineSuffixes(events, context) {
  132. let eventIndex = 0; // Skip first.
  133. while (++eventIndex <= events.length) {
  134. if ((eventIndex === events.length || events[eventIndex][1].type === "lineEnding") && events[eventIndex - 1][1].type === "data") {
  135. const data = events[eventIndex - 1][1];
  136. const chunks = context.sliceStream(data);
  137. let index = chunks.length;
  138. let bufferIndex = -1;
  139. let size = 0;
  140. /** @type {boolean | undefined} */
  141. let tabs;
  142. while (index--) {
  143. const chunk = chunks[index];
  144. if (typeof chunk === 'string') {
  145. bufferIndex = chunk.length;
  146. while (chunk.charCodeAt(bufferIndex - 1) === 32) {
  147. size++;
  148. bufferIndex--;
  149. }
  150. if (bufferIndex) break;
  151. bufferIndex = -1;
  152. }
  153. // Number
  154. else if (chunk === -2) {
  155. tabs = true;
  156. size++;
  157. } else if (chunk === -1) {
  158. // Empty
  159. } else {
  160. // Replacement character, exit.
  161. index++;
  162. break;
  163. }
  164. }
  165. // Allow final trailing whitespace.
  166. if (context._contentTypeTextTrailing && eventIndex === events.length) {
  167. size = 0;
  168. }
  169. if (size) {
  170. const token = {
  171. type: eventIndex === events.length || tabs || size < 2 ? "lineSuffix" : "hardBreakTrailing",
  172. start: {
  173. _bufferIndex: index ? bufferIndex : data.start._bufferIndex + bufferIndex,
  174. _index: data.start._index + index,
  175. line: data.end.line,
  176. column: data.end.column - size,
  177. offset: data.end.offset - size
  178. },
  179. end: {
  180. ...data.end
  181. }
  182. };
  183. data.end = {
  184. ...token.start
  185. };
  186. if (data.start.offset === data.end.offset) {
  187. Object.assign(data, token);
  188. } else {
  189. events.splice(eventIndex, 0, ['enter', token, context], ['exit', token, context]);
  190. eventIndex += 2;
  191. }
  192. }
  193. eventIndex++;
  194. }
  195. }
  196. return events;
  197. }