token.d.ts 836 B

1234567891011121314151617181920212223242526272829303132333435
  1. import { Stack } from "./stack";
  2. export declare class CachedToken {
  3. start: number;
  4. value: number;
  5. end: number;
  6. extended: number;
  7. lookAhead: number;
  8. mask: number;
  9. context: number;
  10. }
  11. export declare class InputStream {
  12. private chunk2;
  13. private chunk2Pos;
  14. next: number;
  15. pos: number;
  16. private rangeIndex;
  17. private range;
  18. peek(offset: number): any;
  19. acceptToken(token: number, endOffset?: number): void;
  20. private getChunk;
  21. private readNext;
  22. advance(n?: number): number;
  23. private setDone;
  24. }
  25. export interface Tokenizer {
  26. }
  27. interface ExternalOptions {
  28. contextual?: boolean;
  29. fallback?: boolean;
  30. extend?: boolean;
  31. }
  32. export declare class ExternalTokenizer {
  33. constructor(token: (input: InputStream, stack: Stack) => void, options?: ExternalOptions);
  34. }
  35. export {};