document.js 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. 'use strict'
  2. Object.defineProperty(exports, '__esModule', {value: true})
  3. var codes = require('../character/codes.js')
  4. var markdownLineEnding = require('../character/markdown-line-ending.js')
  5. var constants = require('../constant/constants.js')
  6. var types = require('../constant/types.js')
  7. var factorySpace = require('../tokenize/factory-space.js')
  8. var partialBlankLine = require('../tokenize/partial-blank-line.js')
  9. var tokenize = initializeDocument
  10. var containerConstruct = {tokenize: tokenizeContainer}
  11. var lazyFlowConstruct = {tokenize: tokenizeLazyFlow}
  12. function initializeDocument(effects) {
  13. var self = this
  14. var stack = []
  15. var continued = 0
  16. var inspectConstruct = {tokenize: tokenizeInspect, partial: true}
  17. var inspectResult
  18. var childFlow
  19. var childToken
  20. return start
  21. function start(code) {
  22. if (continued < stack.length) {
  23. self.containerState = stack[continued][1]
  24. return effects.attempt(
  25. stack[continued][0].continuation,
  26. documentContinue,
  27. documentContinued
  28. )(code)
  29. }
  30. return documentContinued(code)
  31. }
  32. function documentContinue(code) {
  33. continued++
  34. return start(code)
  35. }
  36. function documentContinued(code) {
  37. // If we’re in a concrete construct (such as when expecting another line of
  38. // HTML, or we resulted in lazy content), we can immediately start flow.
  39. if (inspectResult && inspectResult.flowContinue) {
  40. return flowStart(code)
  41. }
  42. self.interrupt =
  43. childFlow &&
  44. childFlow.currentConstruct &&
  45. childFlow.currentConstruct.interruptible
  46. self.containerState = {}
  47. return effects.attempt(
  48. containerConstruct,
  49. containerContinue,
  50. flowStart
  51. )(code)
  52. }
  53. function containerContinue(code) {
  54. stack.push([self.currentConstruct, self.containerState])
  55. self.containerState = undefined
  56. return documentContinued(code)
  57. }
  58. function flowStart(code) {
  59. if (code === codes.eof) {
  60. exitContainers(0, true)
  61. effects.consume(code)
  62. return
  63. }
  64. childFlow = childFlow || self.parser.flow(self.now())
  65. effects.enter(types.chunkFlow, {
  66. contentType: constants.contentTypeFlow,
  67. previous: childToken,
  68. _tokenizer: childFlow
  69. })
  70. return flowContinue(code)
  71. }
  72. function flowContinue(code) {
  73. if (code === codes.eof) {
  74. continueFlow(effects.exit(types.chunkFlow))
  75. return flowStart(code)
  76. }
  77. if (markdownLineEnding(code)) {
  78. effects.consume(code)
  79. continueFlow(effects.exit(types.chunkFlow))
  80. return effects.check(inspectConstruct, documentAfterPeek)
  81. }
  82. effects.consume(code)
  83. return flowContinue
  84. }
  85. function documentAfterPeek(code) {
  86. exitContainers(
  87. inspectResult.continued,
  88. inspectResult && inspectResult.flowEnd
  89. )
  90. continued = 0
  91. return start(code)
  92. }
  93. function continueFlow(token) {
  94. if (childToken) childToken.next = token
  95. childToken = token
  96. childFlow.lazy = inspectResult && inspectResult.lazy
  97. childFlow.defineSkip(token.start)
  98. childFlow.write(self.sliceStream(token))
  99. }
  100. function exitContainers(size, end) {
  101. var index = stack.length
  102. // Close the flow.
  103. if (childFlow && end) {
  104. childFlow.write([codes.eof])
  105. childToken = childFlow = undefined
  106. }
  107. // Exit open containers.
  108. while (index-- > size) {
  109. self.containerState = stack[index][1]
  110. stack[index][0].exit.call(self, effects)
  111. }
  112. stack.length = size
  113. }
  114. function tokenizeInspect(effects, ok) {
  115. var subcontinued = 0
  116. inspectResult = {}
  117. return inspectStart
  118. function inspectStart(code) {
  119. if (subcontinued < stack.length) {
  120. self.containerState = stack[subcontinued][1]
  121. return effects.attempt(
  122. stack[subcontinued][0].continuation,
  123. inspectContinue,
  124. inspectLess
  125. )(code)
  126. }
  127. // If we’re continued but in a concrete flow, we can’t have more
  128. // containers.
  129. if (childFlow.currentConstruct && childFlow.currentConstruct.concrete) {
  130. inspectResult.flowContinue = true
  131. return inspectDone(code)
  132. }
  133. self.interrupt =
  134. childFlow.currentConstruct && childFlow.currentConstruct.interruptible
  135. self.containerState = {}
  136. return effects.attempt(
  137. containerConstruct,
  138. inspectFlowEnd,
  139. inspectDone
  140. )(code)
  141. }
  142. function inspectContinue(code) {
  143. subcontinued++
  144. return self.containerState._closeFlow
  145. ? inspectFlowEnd(code)
  146. : inspectStart(code)
  147. }
  148. function inspectLess(code) {
  149. if (childFlow.currentConstruct && childFlow.currentConstruct.lazy) {
  150. // Maybe another container?
  151. self.containerState = {}
  152. return effects.attempt(
  153. containerConstruct,
  154. inspectFlowEnd,
  155. // Maybe flow, or a blank line?
  156. effects.attempt(
  157. lazyFlowConstruct,
  158. inspectFlowEnd,
  159. effects.check(partialBlankLine, inspectFlowEnd, inspectLazy)
  160. )
  161. )(code)
  162. }
  163. // Otherwise we’re interrupting.
  164. return inspectFlowEnd(code)
  165. }
  166. function inspectLazy(code) {
  167. // Act as if all containers are continued.
  168. subcontinued = stack.length
  169. inspectResult.lazy = true
  170. inspectResult.flowContinue = true
  171. return inspectDone(code)
  172. }
  173. // We’re done with flow if we have more containers, or an interruption.
  174. function inspectFlowEnd(code) {
  175. inspectResult.flowEnd = true
  176. return inspectDone(code)
  177. }
  178. function inspectDone(code) {
  179. inspectResult.continued = subcontinued
  180. self.interrupt = self.containerState = undefined
  181. return ok(code)
  182. }
  183. }
  184. }
  185. function tokenizeContainer(effects, ok, nok) {
  186. return factorySpace(
  187. effects,
  188. effects.attempt(this.parser.constructs.document, ok, nok),
  189. types.linePrefix,
  190. this.parser.constructs.disable.null.indexOf('codeIndented') > -1
  191. ? undefined
  192. : constants.tabSize
  193. )
  194. }
  195. function tokenizeLazyFlow(effects, ok, nok) {
  196. return factorySpace(
  197. effects,
  198. effects.lazy(this.parser.constructs.flow, ok, nok),
  199. types.linePrefix,
  200. this.parser.constructs.disable.null.indexOf('codeIndented') > -1
  201. ? undefined
  202. : constants.tabSize
  203. )
  204. }
  205. exports.tokenize = tokenize