document.mjs 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. export var tokenize = initializeDocument
  2. import codes from '../character/codes.mjs'
  3. import markdownLineEnding from '../character/markdown-line-ending.mjs'
  4. import constants from '../constant/constants.mjs'
  5. import types from '../constant/types.mjs'
  6. import spaceFactory from '../tokenize/factory-space.mjs'
  7. import blank from '../tokenize/partial-blank-line.mjs'
  8. var containerConstruct = {tokenize: tokenizeContainer}
  9. var lazyFlowConstruct = {tokenize: tokenizeLazyFlow}
  10. function initializeDocument(effects) {
  11. var self = this
  12. var stack = []
  13. var continued = 0
  14. var inspectConstruct = {tokenize: tokenizeInspect, partial: true}
  15. var inspectResult
  16. var childFlow
  17. var childToken
  18. return start
  19. function start(code) {
  20. if (continued < stack.length) {
  21. self.containerState = stack[continued][1]
  22. return effects.attempt(
  23. stack[continued][0].continuation,
  24. documentContinue,
  25. documentContinued
  26. )(code)
  27. }
  28. return documentContinued(code)
  29. }
  30. function documentContinue(code) {
  31. continued++
  32. return start(code)
  33. }
  34. function documentContinued(code) {
  35. // If we’re in a concrete construct (such as when expecting another line of
  36. // HTML, or we resulted in lazy content), we can immediately start flow.
  37. if (inspectResult && inspectResult.flowContinue) {
  38. return flowStart(code)
  39. }
  40. self.interrupt =
  41. childFlow &&
  42. childFlow.currentConstruct &&
  43. childFlow.currentConstruct.interruptible
  44. self.containerState = {}
  45. return effects.attempt(
  46. containerConstruct,
  47. containerContinue,
  48. flowStart
  49. )(code)
  50. }
  51. function containerContinue(code) {
  52. stack.push([self.currentConstruct, self.containerState])
  53. self.containerState = undefined
  54. return documentContinued(code)
  55. }
  56. function flowStart(code) {
  57. if (code === codes.eof) {
  58. exitContainers(0, true)
  59. effects.consume(code)
  60. return
  61. }
  62. childFlow = childFlow || self.parser.flow(self.now())
  63. effects.enter(types.chunkFlow, {
  64. contentType: constants.contentTypeFlow,
  65. previous: childToken,
  66. _tokenizer: childFlow
  67. })
  68. return flowContinue(code)
  69. }
  70. function flowContinue(code) {
  71. if (code === codes.eof) {
  72. continueFlow(effects.exit(types.chunkFlow))
  73. return flowStart(code)
  74. }
  75. if (markdownLineEnding(code)) {
  76. effects.consume(code)
  77. continueFlow(effects.exit(types.chunkFlow))
  78. return effects.check(inspectConstruct, documentAfterPeek)
  79. }
  80. effects.consume(code)
  81. return flowContinue
  82. }
  83. function documentAfterPeek(code) {
  84. exitContainers(
  85. inspectResult.continued,
  86. inspectResult && inspectResult.flowEnd
  87. )
  88. continued = 0
  89. return start(code)
  90. }
  91. function continueFlow(token) {
  92. if (childToken) childToken.next = token
  93. childToken = token
  94. childFlow.lazy = inspectResult && inspectResult.lazy
  95. childFlow.defineSkip(token.start)
  96. childFlow.write(self.sliceStream(token))
  97. }
  98. function exitContainers(size, end) {
  99. var index = stack.length
  100. // Close the flow.
  101. if (childFlow && end) {
  102. childFlow.write([codes.eof])
  103. childToken = childFlow = undefined
  104. }
  105. // Exit open containers.
  106. while (index-- > size) {
  107. self.containerState = stack[index][1]
  108. stack[index][0].exit.call(self, effects)
  109. }
  110. stack.length = size
  111. }
  112. function tokenizeInspect(effects, ok) {
  113. var subcontinued = 0
  114. inspectResult = {}
  115. return inspectStart
  116. function inspectStart(code) {
  117. if (subcontinued < stack.length) {
  118. self.containerState = stack[subcontinued][1]
  119. return effects.attempt(
  120. stack[subcontinued][0].continuation,
  121. inspectContinue,
  122. inspectLess
  123. )(code)
  124. }
  125. // If we’re continued but in a concrete flow, we can’t have more
  126. // containers.
  127. if (childFlow.currentConstruct && childFlow.currentConstruct.concrete) {
  128. inspectResult.flowContinue = true
  129. return inspectDone(code)
  130. }
  131. self.interrupt =
  132. childFlow.currentConstruct && childFlow.currentConstruct.interruptible
  133. self.containerState = {}
  134. return effects.attempt(
  135. containerConstruct,
  136. inspectFlowEnd,
  137. inspectDone
  138. )(code)
  139. }
  140. function inspectContinue(code) {
  141. subcontinued++
  142. return self.containerState._closeFlow
  143. ? inspectFlowEnd(code)
  144. : inspectStart(code)
  145. }
  146. function inspectLess(code) {
  147. if (childFlow.currentConstruct && childFlow.currentConstruct.lazy) {
  148. // Maybe another container?
  149. self.containerState = {}
  150. return effects.attempt(
  151. containerConstruct,
  152. inspectFlowEnd,
  153. // Maybe flow, or a blank line?
  154. effects.attempt(
  155. lazyFlowConstruct,
  156. inspectFlowEnd,
  157. effects.check(blank, inspectFlowEnd, inspectLazy)
  158. )
  159. )(code)
  160. }
  161. // Otherwise we’re interrupting.
  162. return inspectFlowEnd(code)
  163. }
  164. function inspectLazy(code) {
  165. // Act as if all containers are continued.
  166. subcontinued = stack.length
  167. inspectResult.lazy = true
  168. inspectResult.flowContinue = true
  169. return inspectDone(code)
  170. }
  171. // We’re done with flow if we have more containers, or an interruption.
  172. function inspectFlowEnd(code) {
  173. inspectResult.flowEnd = true
  174. return inspectDone(code)
  175. }
  176. function inspectDone(code) {
  177. inspectResult.continued = subcontinued
  178. self.interrupt = self.containerState = undefined
  179. return ok(code)
  180. }
  181. }
  182. }
  183. function tokenizeContainer(effects, ok, nok) {
  184. return spaceFactory(
  185. effects,
  186. effects.attempt(this.parser.constructs.document, ok, nok),
  187. types.linePrefix,
  188. this.parser.constructs.disable.null.indexOf('codeIndented') > -1
  189. ? undefined
  190. : constants.tabSize
  191. )
  192. }
  193. function tokenizeLazyFlow(effects, ok, nok) {
  194. return spaceFactory(
  195. effects,
  196. effects.lazy(this.parser.constructs.flow, ok, nok),
  197. types.linePrefix,
  198. this.parser.constructs.disable.null.indexOf('codeIndented') > -1
  199. ? undefined
  200. : constants.tabSize
  201. )
  202. }