Fixes microsoft#134973: Cast to uint32 before validating that semantic tokens do not overlap

Author: alexdima

src/vs/editor/common/services/semanticTokensProviderStyling.ts

@@ -20,9 +20,9 @@ export class SemanticTokensProviderStyling {
 
 	constructor(
 		private readonly _legend: SemanticTokensLegend,
-		private readonly _themeService: IThemeService,
-		private readonly _modeService: IModeService,
-		private readonly _logService: ILogService
+		@IThemeService private readonly _themeService: IThemeService,
+		@IModeService private readonly _modeService: IModeService,
+		@ILogService private readonly _logService: ILogService
 	) {
 		this._hashTable = new HashTable();
 		this._hasWarnedOverlappingTokens = false;
@@ -162,8 +162,10 @@ export function toMultilineTokens2(tokens: SemanticTokens, styling: SemanticToke
 			const srcOffset = 5 * tokenIndex;
 			const deltaLine = srcData[srcOffset];
 			const deltaCharacter = srcData[srcOffset + 1];
-			const lineNumber = lastLineNumber + deltaLine;
-			const startCharacter = (deltaLine === 0 ? lastStartCharacter + deltaCharacter : deltaCharacter);
+			// Casting both `lineNumber` and `startCharacter` here to uint32 using `|0`
+			// to do checks below with the actual value that will be inserted in the Uint32Array result
+			const lineNumber = (lastLineNumber + deltaLine) | 0;
+			const startCharacter = (deltaLine === 0 ? (lastStartCharacter + deltaCharacter) | 0 : deltaCharacter);
 			const length = srcData[srcOffset + 2];
 			const tokenTypeIndex = srcData[srcOffset + 3];
 			const tokenModifierSet = srcData[srcOffset + 4];

src/vs/editor/test/common/editorTestUtils.ts

@@ -18,7 +18,6 @@ import { IDialogService } from 'vs/platform/dialogs/common/dialogs';
 import { TestDialogService } from 'vs/platform/dialogs/test/common/testDialogService';
 import { SyncDescriptor } from 'vs/platform/instantiation/common/descriptors';
 import { IInstantiationService, ServiceIdentifier } from 'vs/platform/instantiation/common/instantiation';
-import { InstantiationService } from 'vs/platform/instantiation/common/instantiationService';
 import { ServiceCollection } from 'vs/platform/instantiation/common/serviceCollection';
 import { ILogService, NullLogService } from 'vs/platform/log/common/log';
 import { INotificationService } from 'vs/platform/notification/common/notification';
@@ -30,6 +29,7 @@ import { UndoRedoService } from 'vs/platform/undoRedo/common/undoRedoService';
 import { TestTextResourcePropertiesService } from 'vs/editor/test/common/services/testTextResourcePropertiesService';
 import { IModelService } from 'vs/editor/common/services/modelService';
 import { ModelServiceImpl } from 'vs/editor/common/services/modelServiceImpl';
+import { TestInstantiationService } from 'vs/platform/instantiation/test/common/instantiationServiceMock';
 
 class TestTextModel extends TextModel {
 	public registerDisposable(disposable: IDisposable): void {
@@ -81,7 +81,7 @@ export function createTextModel2(instantiationService: IInstantiationService, te
 	return instantiationService.createInstance(TestTextModel, text, options, languageId, uri);
 }
 
-export function createModelServices(services: ServiceCollection = new ServiceCollection()): [IInstantiationService, DisposableStore] {
+export function createModelServices(services: ServiceCollection = new ServiceCollection()): [TestInstantiationService, DisposableStore] {
 	const serviceIdentifiers: ServiceIdentifier<any>[] = [];
 	const define = <T>(id: ServiceIdentifier<T>, ctor: new (...args: any[]) => T) => {
 		if (!services.has(id)) {
@@ -101,7 +101,7 @@ export function createModelServices(services: ServiceCollection = new ServiceCol
 	define(ILogService, NullLogService);
 	define(IModelService, ModelServiceImpl);
 
-	const instantiationService: IInstantiationService = new InstantiationService(services);
+	const instantiationService = new TestInstantiationService(services);
 	const disposables = new DisposableStore();
 	disposables.add(toDisposable(() => {
 		for (const id of serviceIdentifiers) {

src/vs/editor/test/common/services/semanticTokensProviderStyling.test.ts

@@ -0,0 +1,75 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as assert from 'assert';
+import { DisposableStore } from 'vs/base/common/lifecycle';
+import { MultilineTokens2, SparseEncodedTokens } from 'vs/editor/common/model/tokensStore';
+import { MetadataConsts } from 'vs/editor/common/modes';
+import { ModesRegistry } from 'vs/editor/common/modes/modesRegistry';
+import { SemanticTokensProviderStyling, toMultilineTokens2 } from 'vs/editor/common/services/semanticTokensProviderStyling';
+import { createModelServices } from 'vs/editor/test/common/editorTestUtils';
+import { TestInstantiationService } from 'vs/platform/instantiation/test/common/instantiationServiceMock';
+import { IThemeService, ITokenStyle } from 'vs/platform/theme/common/themeService';
+
+suite('ModelService', () => {
+	let disposables: DisposableStore;
+	let instantiationService: TestInstantiationService;
+
+	setup(() => {
+		[instantiationService, disposables] = createModelServices();
+	});
+
+	teardown(() => {
+		disposables.dispose();
+	});
+
+	test('issue #134973: invalid semantic tokens should be handled better', () => {
+		const languageId = 'java';
+		disposables.add(ModesRegistry.registerLanguage({ id: languageId }));
+		const legend = {
+			tokenTypes: ['st0', 'st1', 'st2', 'st3', 'st4', 'st5', 'st6', 'st7', 'st8', 'st9', 'st10'],
+			tokenModifiers: []
+		};
+		instantiationService.stub(IThemeService, <Partial<IThemeService>>{
+			getColorTheme() {
+				return {
+					getTokenStyleMetadata: (tokenType, tokenModifiers, languageId): ITokenStyle => {
+						return {
+							foreground: parseInt(tokenType.substr(2), 10)
+						};
+					}
+				};
+			}
+		});
+		const styling = instantiationService.createInstance(SemanticTokensProviderStyling, legend);
+		const badTokens = {
+			data: new Uint32Array([
+				0, 13, 16, 1, 0,
+				1, 2, 6, 2, 0,
+				0, 7, 6, 3, 0,
+				0, 15, 8, 4, 0,
+				0, 17, 1, 5, 0,
+				0, 7, 5, 6, 0,
+				1, 12, 8, 7, 0,
+				0, 19, 5, 8, 0,
+				0, 7, 1, 9, 0,
+				0, 4294967294, 5, 10, 0
+			])
+		};
+		const result = toMultilineTokens2(badTokens, styling, languageId);
+		const expected = new MultilineTokens2(1, new SparseEncodedTokens(new Uint32Array([
+			0, 13, 29, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (1 << MetadataConsts.FOREGROUND_OFFSET)),
+			1, 2, 8, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (2 << MetadataConsts.FOREGROUND_OFFSET)),
+			1, 9, 15, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (3 << MetadataConsts.FOREGROUND_OFFSET)),
+			1, 24, 32, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (4 << MetadataConsts.FOREGROUND_OFFSET)),
+			1, 41, 42, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (5 << MetadataConsts.FOREGROUND_OFFSET)),
+			1, 48, 53, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (6 << MetadataConsts.FOREGROUND_OFFSET)),
+			2, 12, 20, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (7 << MetadataConsts.FOREGROUND_OFFSET)),
+			2, 31, 36, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (8 << MetadataConsts.FOREGROUND_OFFSET)),
+			2, 36, 41, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (9 << MetadataConsts.FOREGROUND_OFFSET)),
+		])));
+		assert.deepStrictEqual(result.toString(), expected.toString());
+	});
+});