textToHtmlTokenizer.js 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119
  1. /*---------------------------------------------------------------------------------------------
  2. * Copyright (c) Microsoft Corporation. All rights reserved.
  3. * Licensed under the MIT License. See License.txt in the project root for license information.
  4. *--------------------------------------------------------------------------------------------*/
  5. import * as strings from '../../../base/common/strings.js';
  6. import { LineTokens } from '../core/lineTokens.js';
  7. import { NULL_STATE, nullTokenize2 } from './nullMode.js';
  8. const fallback = {
  9. getInitialState: () => NULL_STATE,
  10. tokenize2: (buffer, hasEOL, state, deltaOffset) => nullTokenize2(0 /* Null */, buffer, state, deltaOffset)
  11. };
  12. export function tokenizeToString(text, languageIdCodec, tokenizationSupport = fallback) {
  13. return _tokenizeToString(text, languageIdCodec, tokenizationSupport || fallback);
  14. }
  15. export function tokenizeLineToHTML(text, viewLineTokens, colorMap, startOffset, endOffset, tabSize, useNbsp) {
  16. let result = `<div>`;
  17. let charIndex = startOffset;
  18. let tabsCharDelta = 0;
  19. let prevIsSpace = true;
  20. for (let tokenIndex = 0, tokenCount = viewLineTokens.getCount(); tokenIndex < tokenCount; tokenIndex++) {
  21. const tokenEndIndex = viewLineTokens.getEndOffset(tokenIndex);
  22. if (tokenEndIndex <= startOffset) {
  23. continue;
  24. }
  25. let partContent = '';
  26. for (; charIndex < tokenEndIndex && charIndex < endOffset; charIndex++) {
  27. const charCode = text.charCodeAt(charIndex);
  28. switch (charCode) {
  29. case 9 /* Tab */:
  30. let insertSpacesCount = tabSize - (charIndex + tabsCharDelta) % tabSize;
  31. tabsCharDelta += insertSpacesCount - 1;
  32. while (insertSpacesCount > 0) {
  33. if (useNbsp && prevIsSpace) {
  34. partContent += '&#160;';
  35. prevIsSpace = false;
  36. }
  37. else {
  38. partContent += ' ';
  39. prevIsSpace = true;
  40. }
  41. insertSpacesCount--;
  42. }
  43. break;
  44. case 60 /* LessThan */:
  45. partContent += '&lt;';
  46. prevIsSpace = false;
  47. break;
  48. case 62 /* GreaterThan */:
  49. partContent += '&gt;';
  50. prevIsSpace = false;
  51. break;
  52. case 38 /* Ampersand */:
  53. partContent += '&amp;';
  54. prevIsSpace = false;
  55. break;
  56. case 0 /* Null */:
  57. partContent += '&#00;';
  58. prevIsSpace = false;
  59. break;
  60. case 65279 /* UTF8_BOM */:
  61. case 8232 /* LINE_SEPARATOR */:
  62. case 8233 /* PARAGRAPH_SEPARATOR */:
  63. case 133 /* NEXT_LINE */:
  64. partContent += '\ufffd';
  65. prevIsSpace = false;
  66. break;
  67. case 13 /* CarriageReturn */:
  68. // zero width space, because carriage return would introduce a line break
  69. partContent += '&#8203';
  70. prevIsSpace = false;
  71. break;
  72. case 32 /* Space */:
  73. if (useNbsp && prevIsSpace) {
  74. partContent += '&#160;';
  75. prevIsSpace = false;
  76. }
  77. else {
  78. partContent += ' ';
  79. prevIsSpace = true;
  80. }
  81. break;
  82. default:
  83. partContent += String.fromCharCode(charCode);
  84. prevIsSpace = false;
  85. }
  86. }
  87. result += `<span style="${viewLineTokens.getInlineStyle(tokenIndex, colorMap)}">${partContent}</span>`;
  88. if (tokenEndIndex > endOffset || charIndex >= endOffset) {
  89. break;
  90. }
  91. }
  92. result += `</div>`;
  93. return result;
  94. }
  95. function _tokenizeToString(text, languageIdCodec, tokenizationSupport) {
  96. let result = `<div class="monaco-tokenized-source">`;
  97. const lines = strings.splitLines(text);
  98. let currentState = tokenizationSupport.getInitialState();
  99. for (let i = 0, len = lines.length; i < len; i++) {
  100. const line = lines[i];
  101. if (i > 0) {
  102. result += `<br/>`;
  103. }
  104. const tokenizationResult = tokenizationSupport.tokenize2(line, true, currentState, 0);
  105. LineTokens.convertToEndOffset(tokenizationResult.tokens, line.length);
  106. const lineTokens = new LineTokens(tokenizationResult.tokens, line, languageIdCodec);
  107. const viewLineTokens = lineTokens.inflate();
  108. let startOffset = 0;
  109. for (let j = 0, lenJ = viewLineTokens.getCount(); j < lenJ; j++) {
  110. const type = viewLineTokens.getClassName(j);
  111. const endIndex = viewLineTokens.getEndOffset(j);
  112. result += `<span class="${type}">${strings.escape(line.substring(startOffset, endIndex))}</span>`;
  113. startOffset = endIndex;
  114. }
  115. currentState = tokenizationResult.endState;
  116. }
  117. result += `</div>`;
  118. return result;
  119. }