subtokenize.js 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219
  1. 'use strict'
  2. var assert = require('assert')
  3. var codes = require('../character/codes.js')
  4. var assign = require('../constant/assign.js')
  5. var types = require('../constant/types.js')
  6. var chunkedSplice = require('./chunked-splice.js')
  7. var shallow = require('./shallow.js')
  8. function _interopDefaultLegacy(e) {
  9. return e && typeof e === 'object' && 'default' in e ? e : {default: e}
  10. }
  11. var assert__default = /*#__PURE__*/ _interopDefaultLegacy(assert)
  12. function subtokenize(events) {
  13. var jumps = {}
  14. var index = -1
  15. var event
  16. var lineIndex
  17. var otherIndex
  18. var otherEvent
  19. var parameters
  20. var subevents
  21. var more
  22. while (++index < events.length) {
  23. while (index in jumps) {
  24. index = jumps[index]
  25. }
  26. event = events[index]
  27. // Add a hook for the GFM tasklist extension, which needs to know if text
  28. // is in the first content of a list item.
  29. if (
  30. index &&
  31. event[1].type === types.chunkFlow &&
  32. events[index - 1][1].type === types.listItemPrefix
  33. ) {
  34. subevents = event[1]._tokenizer.events
  35. otherIndex = 0
  36. if (
  37. otherIndex < subevents.length &&
  38. subevents[otherIndex][1].type === types.lineEndingBlank
  39. ) {
  40. otherIndex += 2
  41. }
  42. if (
  43. otherIndex < subevents.length &&
  44. subevents[otherIndex][1].type === types.content
  45. ) {
  46. while (++otherIndex < subevents.length) {
  47. if (subevents[otherIndex][1].type === types.content) {
  48. break
  49. }
  50. if (subevents[otherIndex][1].type === types.chunkText) {
  51. subevents[otherIndex][1].isInFirstContentOfListItem = true
  52. otherIndex++
  53. }
  54. }
  55. }
  56. }
  57. // Enter.
  58. if (event[0] === 'enter') {
  59. if (event[1].contentType) {
  60. assign(jumps, subcontent(events, index))
  61. index = jumps[index]
  62. more = true
  63. }
  64. }
  65. // Exit.
  66. else if (event[1]._container || event[1]._movePreviousLineEndings) {
  67. otherIndex = index
  68. lineIndex = undefined
  69. while (otherIndex--) {
  70. otherEvent = events[otherIndex]
  71. if (
  72. otherEvent[1].type === types.lineEnding ||
  73. otherEvent[1].type === types.lineEndingBlank
  74. ) {
  75. if (otherEvent[0] === 'enter') {
  76. if (lineIndex) {
  77. events[lineIndex][1].type = types.lineEndingBlank
  78. }
  79. otherEvent[1].type = types.lineEnding
  80. lineIndex = otherIndex
  81. }
  82. } else {
  83. break
  84. }
  85. }
  86. if (lineIndex) {
  87. // Fix position.
  88. event[1].end = shallow(events[lineIndex][1].start)
  89. // Switch container exit w/ line endings.
  90. parameters = events.slice(lineIndex, index)
  91. parameters.unshift(event)
  92. chunkedSplice(events, lineIndex, index - lineIndex + 1, parameters)
  93. }
  94. }
  95. }
  96. return !more
  97. }
  98. function subcontent(events, eventIndex) {
  99. var token = events[eventIndex][1]
  100. var context = events[eventIndex][2]
  101. var startPosition = eventIndex - 1
  102. var startPositions = []
  103. var tokenizer =
  104. token._tokenizer || context.parser[token.contentType](token.start)
  105. var childEvents = tokenizer.events
  106. var jumps = []
  107. var gaps = {}
  108. var stream
  109. var previous
  110. var index
  111. var entered
  112. var end
  113. var adjust
  114. // Loop forward through the linked tokens to pass them in order to the
  115. // subtokenizer.
  116. while (token) {
  117. // Find the position of the event for this token.
  118. while (events[++startPosition][1] !== token) {
  119. // Empty.
  120. }
  121. startPositions.push(startPosition)
  122. if (!token._tokenizer) {
  123. stream = context.sliceStream(token)
  124. if (!token.next) {
  125. stream.push(codes.eof)
  126. }
  127. if (previous) {
  128. tokenizer.defineSkip(token.start)
  129. }
  130. if (token.isInFirstContentOfListItem) {
  131. tokenizer._gfmTasklistFirstContentOfListItem = true
  132. }
  133. tokenizer.write(stream)
  134. if (token.isInFirstContentOfListItem) {
  135. tokenizer._gfmTasklistFirstContentOfListItem = undefined
  136. }
  137. }
  138. // Unravel the next token.
  139. previous = token
  140. token = token.next
  141. }
  142. // Now, loop back through all events (and linked tokens), to figure out which
  143. // parts belong where.
  144. token = previous
  145. index = childEvents.length
  146. while (index--) {
  147. // Make sure we’ve at least seen something (final eol is part of the last
  148. // token).
  149. if (childEvents[index][0] === 'enter') {
  150. entered = true
  151. } else if (
  152. // Find a void token that includes a break.
  153. entered &&
  154. childEvents[index][1].type === childEvents[index - 1][1].type &&
  155. childEvents[index][1].start.line !== childEvents[index][1].end.line
  156. ) {
  157. add(childEvents.slice(index + 1, end))
  158. assert__default['default'](token.previous, 'expected a previous token')
  159. // Help GC.
  160. token._tokenizer = token.next = undefined
  161. token = token.previous
  162. end = index + 1
  163. }
  164. }
  165. assert__default['default'](!token.previous, 'expected no previous token')
  166. // Help GC.
  167. tokenizer.events = token._tokenizer = token.next = undefined
  168. // Do head:
  169. add(childEvents.slice(0, end))
  170. index = -1
  171. adjust = 0
  172. while (++index < jumps.length) {
  173. gaps[adjust + jumps[index][0]] = adjust + jumps[index][1]
  174. adjust += jumps[index][1] - jumps[index][0] - 1
  175. }
  176. return gaps
  177. function add(slice) {
  178. var start = startPositions.pop()
  179. jumps.unshift([start, start + slice.length - 1])
  180. chunkedSplice(events, start, 2, slice)
  181. }
  182. }
  183. module.exports = subtokenize