semanticTokensProviderStyling.js 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259
  1. /*---------------------------------------------------------------------------------------------
  2. * Copyright (c) Microsoft Corporation. All rights reserved.
  3. * Licensed under the MIT License. See License.txt in the project root for license information.
  4. *--------------------------------------------------------------------------------------------*/
  5. var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
  6. var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
  7. if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
  8. else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
  9. return c > 3 && r && Object.defineProperty(target, key, r), r;
  10. };
  11. var __param = (this && this.__param) || function (paramIndex, decorator) {
  12. return function (target, key) { decorator(target, key, paramIndex); }
  13. };
  14. import { TokenMetadata } from '../modes.js';
  15. import { IThemeService } from '../../../platform/theme/common/themeService.js';
  16. import { ILogService, LogLevel } from '../../../platform/log/common/log.js';
  17. import { MultilineTokens2, SparseEncodedTokens } from '../model/tokensStore.js';
  18. import { IModeService } from './modeService.js';
  19. let SemanticTokensProviderStyling = class SemanticTokensProviderStyling {
  20. constructor(_legend, _themeService, _modeService, _logService) {
  21. this._legend = _legend;
  22. this._themeService = _themeService;
  23. this._modeService = _modeService;
  24. this._logService = _logService;
  25. this._hashTable = new HashTable();
  26. this._hasWarnedOverlappingTokens = false;
  27. }
  28. getMetadata(tokenTypeIndex, tokenModifierSet, languageId) {
  29. const encodedLanguageId = this._modeService.languageIdCodec.encodeLanguageId(languageId);
  30. const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet, encodedLanguageId);
  31. let metadata;
  32. if (entry) {
  33. metadata = entry.metadata;
  34. if (this._logService.getLevel() === LogLevel.Trace) {
  35. this._logService.trace(`SemanticTokensProviderStyling [CACHED] ${tokenTypeIndex} / ${tokenModifierSet}: foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`);
  36. }
  37. }
  38. else {
  39. let tokenType = this._legend.tokenTypes[tokenTypeIndex];
  40. const tokenModifiers = [];
  41. if (tokenType) {
  42. let modifierSet = tokenModifierSet;
  43. for (let modifierIndex = 0; modifierSet > 0 && modifierIndex < this._legend.tokenModifiers.length; modifierIndex++) {
  44. if (modifierSet & 1) {
  45. tokenModifiers.push(this._legend.tokenModifiers[modifierIndex]);
  46. }
  47. modifierSet = modifierSet >> 1;
  48. }
  49. if (modifierSet > 0 && this._logService.getLevel() === LogLevel.Trace) {
  50. this._logService.trace(`SemanticTokensProviderStyling: unknown token modifier index: ${tokenModifierSet.toString(2)} for legend: ${JSON.stringify(this._legend.tokenModifiers)}`);
  51. tokenModifiers.push('not-in-legend');
  52. }
  53. const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers, languageId);
  54. if (typeof tokenStyle === 'undefined') {
  55. metadata = 2147483647 /* NO_STYLING */;
  56. }
  57. else {
  58. metadata = 0;
  59. if (typeof tokenStyle.italic !== 'undefined') {
  60. const italicBit = (tokenStyle.italic ? 1 /* Italic */ : 0) << 11 /* FONT_STYLE_OFFSET */;
  61. metadata |= italicBit | 1 /* SEMANTIC_USE_ITALIC */;
  62. }
  63. if (typeof tokenStyle.bold !== 'undefined') {
  64. const boldBit = (tokenStyle.bold ? 2 /* Bold */ : 0) << 11 /* FONT_STYLE_OFFSET */;
  65. metadata |= boldBit | 2 /* SEMANTIC_USE_BOLD */;
  66. }
  67. if (typeof tokenStyle.underline !== 'undefined') {
  68. const underlineBit = (tokenStyle.underline ? 4 /* Underline */ : 0) << 11 /* FONT_STYLE_OFFSET */;
  69. metadata |= underlineBit | 4 /* SEMANTIC_USE_UNDERLINE */;
  70. }
  71. if (tokenStyle.foreground) {
  72. const foregroundBits = (tokenStyle.foreground) << 14 /* FOREGROUND_OFFSET */;
  73. metadata |= foregroundBits | 8 /* SEMANTIC_USE_FOREGROUND */;
  74. }
  75. if (metadata === 0) {
  76. // Nothing!
  77. metadata = 2147483647 /* NO_STYLING */;
  78. }
  79. }
  80. }
  81. else {
  82. if (this._logService.getLevel() === LogLevel.Trace) {
  83. this._logService.trace(`SemanticTokensProviderStyling: unknown token type index: ${tokenTypeIndex} for legend: ${JSON.stringify(this._legend.tokenTypes)}`);
  84. }
  85. metadata = 2147483647 /* NO_STYLING */;
  86. tokenType = 'not-in-legend';
  87. }
  88. this._hashTable.add(tokenTypeIndex, tokenModifierSet, encodedLanguageId, metadata);
  89. if (this._logService.getLevel() === LogLevel.Trace) {
  90. this._logService.trace(`SemanticTokensProviderStyling ${tokenTypeIndex} (${tokenType}) / ${tokenModifierSet} (${tokenModifiers.join(' ')}): foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`);
  91. }
  92. }
  93. return metadata;
  94. }
  95. warnOverlappingSemanticTokens(lineNumber, startColumn) {
  96. if (!this._hasWarnedOverlappingTokens) {
  97. this._hasWarnedOverlappingTokens = true;
  98. console.warn(`Overlapping semantic tokens detected at lineNumber ${lineNumber}, column ${startColumn}`);
  99. }
  100. }
  101. };
  102. SemanticTokensProviderStyling = __decorate([
  103. __param(1, IThemeService),
  104. __param(2, IModeService),
  105. __param(3, ILogService)
  106. ], SemanticTokensProviderStyling);
  107. export { SemanticTokensProviderStyling };
  108. export function toMultilineTokens2(tokens, styling, languageId) {
  109. const srcData = tokens.data;
  110. const tokenCount = (tokens.data.length / 5) | 0;
  111. const tokensPerArea = Math.max(Math.ceil(tokenCount / 1024 /* DesiredMaxAreas */), 400 /* DesiredTokensPerArea */);
  112. const result = [];
  113. let tokenIndex = 0;
  114. let lastLineNumber = 1;
  115. let lastStartCharacter = 0;
  116. while (tokenIndex < tokenCount) {
  117. const tokenStartIndex = tokenIndex;
  118. let tokenEndIndex = Math.min(tokenStartIndex + tokensPerArea, tokenCount);
  119. // Keep tokens on the same line in the same area...
  120. if (tokenEndIndex < tokenCount) {
  121. let smallTokenEndIndex = tokenEndIndex;
  122. while (smallTokenEndIndex - 1 > tokenStartIndex && srcData[5 * smallTokenEndIndex] === 0) {
  123. smallTokenEndIndex--;
  124. }
  125. if (smallTokenEndIndex - 1 === tokenStartIndex) {
  126. // there are so many tokens on this line that our area would be empty, we must now go right
  127. let bigTokenEndIndex = tokenEndIndex;
  128. while (bigTokenEndIndex + 1 < tokenCount && srcData[5 * bigTokenEndIndex] === 0) {
  129. bigTokenEndIndex++;
  130. }
  131. tokenEndIndex = bigTokenEndIndex;
  132. }
  133. else {
  134. tokenEndIndex = smallTokenEndIndex;
  135. }
  136. }
  137. let destData = new Uint32Array((tokenEndIndex - tokenStartIndex) * 4);
  138. let destOffset = 0;
  139. let areaLine = 0;
  140. let prevLineNumber = 0;
  141. let prevStartCharacter = 0;
  142. let prevEndCharacter = 0;
  143. while (tokenIndex < tokenEndIndex) {
  144. const srcOffset = 5 * tokenIndex;
  145. const deltaLine = srcData[srcOffset];
  146. const deltaCharacter = srcData[srcOffset + 1];
  147. // Casting both `lineNumber` and `startCharacter` here to uint32 using `|0`
  148. // to do checks below with the actual value that will be inserted in the Uint32Array result
  149. const lineNumber = (lastLineNumber + deltaLine) | 0;
  150. const startCharacter = (deltaLine === 0 ? (lastStartCharacter + deltaCharacter) | 0 : deltaCharacter);
  151. const length = srcData[srcOffset + 2];
  152. const tokenTypeIndex = srcData[srcOffset + 3];
  153. const tokenModifierSet = srcData[srcOffset + 4];
  154. const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet, languageId);
  155. if (metadata !== 2147483647 /* NO_STYLING */) {
  156. if (areaLine === 0) {
  157. areaLine = lineNumber;
  158. }
  159. if (prevLineNumber === lineNumber && prevEndCharacter > startCharacter) {
  160. styling.warnOverlappingSemanticTokens(lineNumber, startCharacter + 1);
  161. if (prevStartCharacter < startCharacter) {
  162. // the previous token survives after the overlapping one
  163. destData[destOffset - 4 + 2] = startCharacter;
  164. }
  165. else {
  166. // the previous token is entirely covered by the overlapping one
  167. destOffset -= 4;
  168. }
  169. }
  170. destData[destOffset] = lineNumber - areaLine;
  171. destData[destOffset + 1] = startCharacter;
  172. destData[destOffset + 2] = startCharacter + length;
  173. destData[destOffset + 3] = metadata;
  174. destOffset += 4;
  175. prevLineNumber = lineNumber;
  176. prevStartCharacter = startCharacter;
  177. prevEndCharacter = startCharacter + length;
  178. }
  179. lastLineNumber = lineNumber;
  180. lastStartCharacter = startCharacter;
  181. tokenIndex++;
  182. }
  183. if (destOffset !== destData.length) {
  184. destData = destData.subarray(0, destOffset);
  185. }
  186. const tokens = new MultilineTokens2(areaLine, new SparseEncodedTokens(destData));
  187. result.push(tokens);
  188. }
  189. return result;
  190. }
  191. class HashTableEntry {
  192. constructor(tokenTypeIndex, tokenModifierSet, languageId, metadata) {
  193. this.tokenTypeIndex = tokenTypeIndex;
  194. this.tokenModifierSet = tokenModifierSet;
  195. this.languageId = languageId;
  196. this.metadata = metadata;
  197. this.next = null;
  198. }
  199. }
  200. class HashTable {
  201. constructor() {
  202. this._elementsCount = 0;
  203. this._currentLengthIndex = 0;
  204. this._currentLength = HashTable._SIZES[this._currentLengthIndex];
  205. this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0);
  206. this._elements = [];
  207. HashTable._nullOutEntries(this._elements, this._currentLength);
  208. }
  209. static _nullOutEntries(entries, length) {
  210. for (let i = 0; i < length; i++) {
  211. entries[i] = null;
  212. }
  213. }
  214. _hash2(n1, n2) {
  215. return (((n1 << 5) - n1) + n2) | 0; // n1 * 31 + n2, keep as int32
  216. }
  217. _hashFunc(tokenTypeIndex, tokenModifierSet, languageId) {
  218. return this._hash2(this._hash2(tokenTypeIndex, tokenModifierSet), languageId) % this._currentLength;
  219. }
  220. get(tokenTypeIndex, tokenModifierSet, languageId) {
  221. const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet, languageId);
  222. let p = this._elements[hash];
  223. while (p) {
  224. if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet && p.languageId === languageId) {
  225. return p;
  226. }
  227. p = p.next;
  228. }
  229. return null;
  230. }
  231. add(tokenTypeIndex, tokenModifierSet, languageId, metadata) {
  232. this._elementsCount++;
  233. if (this._growCount !== 0 && this._elementsCount >= this._growCount) {
  234. // expand!
  235. const oldElements = this._elements;
  236. this._currentLengthIndex++;
  237. this._currentLength = HashTable._SIZES[this._currentLengthIndex];
  238. this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0);
  239. this._elements = [];
  240. HashTable._nullOutEntries(this._elements, this._currentLength);
  241. for (const first of oldElements) {
  242. let p = first;
  243. while (p) {
  244. const oldNext = p.next;
  245. p.next = null;
  246. this._add(p);
  247. p = oldNext;
  248. }
  249. }
  250. }
  251. this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, languageId, metadata));
  252. }
  253. _add(element) {
  254. const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet, element.languageId);
  255. element.next = this._elements[hash];
  256. this._elements[hash] = element;
  257. }
  258. }
  259. HashTable._SIZES = [3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381, 32749, 65521, 131071, 262139, 524287, 1048573, 2097143];