Fixed emoji filtering and improved the filtering performance.

.eslintrc.js

@@ -17,7 +17,8 @@ module.exports = {
 		// The CKEditor 5 core DLL build is created from JavaScript files.
 		// ESLint should not process compiled TypeScript.
 		'src/*.js',
-		'**/*.d.ts'
+		'**/*.d.ts',
+		'packages/ckeditor5-emoji/src/utils/isemojisupported.ts'
 	],
 	rules: {
 		'ckeditor5-rules/ckeditor-imports': 'error',

LICENSE.md

@@ -23,6 +23,7 @@ The following libraries are included in CKEditor under the [MIT license](https:/
 * color-parse - Copyright (c) 2015 Dmitry Ivanov.
 * emoji-picker-element-data - Copyright (c) 2020 Nolan Lawson.
 * Fuse.js - Copyright (c) 2017 Kirollos Risk.
+* is-emoji-supported - Copyright (c) 2023, Koala Interactive SAS
 * Lodash - Copyright (c) JS Foundation and other contributors https://js.foundation/. Based on Underscore.js, copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors http://underscorejs.org/.
 * Marked - Copyright (c) 2018+, MarkedJS (https://github.com/markedjs/), copyright (c) 2011–2018, Christopher Jeffrey (https://github.com/chjj/).
 * Turndown - Copyright (c) 2017+ Dom Christie.

packages/ckeditor5-emoji/LICENSE.md

@@ -20,6 +20,7 @@ The following libraries are included in CKEditor 5 emoji feature under the
 
 * emoji-picker-element-data - Copyright (c) 2020 Nolan Lawson.
 * Fuse.js - Copyright (c) 2017 Kirollos Risk.
+* is-emoji-supported - Copyright (c) 2023, Koala Interactive SAS
 * Lodash - Copyright (c) JS Foundation and other contributors https://js.foundation/. Based on Underscore.js, copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors http://underscorejs.org/.
 
 Trademarks

packages/ckeditor5-emoji/src/emojirepository.ts

@@ -10,9 +10,10 @@
 import Fuse from 'fuse.js';
 import { groupBy } from 'lodash-es';
 
-import { Plugin, type Editor } from 'ckeditor5/src/core.js';
+import { type Editor, Plugin } from 'ckeditor5/src/core.js';
 import { logWarning } from 'ckeditor5/src/utils.js';
 import type { SkinToneId } from './emojiconfig.js';
+import isEmojiSupported from './utils/isemojisupported.js';
 
 // An endpoint from which the emoji database will be downloaded during plugin initialization.
 // The `{version}` placeholder is replaced with the value from editor config.
@@ -29,6 +30,16 @@ const SKIN_TONE_MAP: Record<number, SkinToneId> = {
 
 const BASELINE_EMOJI_WIDTH = 24;
 
+/**
+ * Object storing emoji support level by the operating system.
+ * Some emojis that are perceived as supported, are compound-emoji from previous version. For those emojis
+ * there has to be performed an additional width check to verify system support.
+ */
+const EMOJI_SUPPORT_LEVEL = {
+	'🫩': 16, // face with bags under eyes
+	'🫨': 15 // shaking head
+};
+
 /**
  * The emoji repository plugin.
  *
@@ -96,6 +107,7 @@ export default class EmojiRepository extends Plugin {
 		const emojiVersion = this.editor.config.get( 'emoji.version' )!;
 		const emojiDatabaseUrl = EMOJI_DATABASE_URL.replace( '{version}', `${ emojiVersion }` );
 		const emojiDatabase = await loadEmojiDatabase( emojiDatabaseUrl );
+		const emojiSupportedVersionByOs = EmojiRepository._getEmojiSupportedVersionByOs();
 
 		// Skip the initialization if the emoji database download has failed.
 		// An empty database prevents the initialization of other dependent plugins, such as `EmojiMention` and `EmojiPicker`.
@@ -108,7 +120,8 @@ export default class EmojiRepository extends Plugin {
 		// Store the emoji database after normalizing the raw data.
 		this._database = emojiDatabase
 			.filter( item => isEmojiCategoryAllowed( item ) )
-			.filter( item => EmojiRepository._isEmojiSupported( item, container ) )
+			.filter( item => item.version <= emojiSupportedVersionByOs )
+			.filter( item => !EmojiRepository._hasZwj( item.emoji ) ? true : EmojiRepository._isEmojiZwjSupported( item, container ) )
 			.map( item => normalizeEmojiSkinTone( item ) );
 
 		container.remove();
@@ -226,11 +239,25 @@ export default class EmojiRepository extends Plugin {
 	}
 
 	/**
-	 * A function used to check if the given emoji is supported in the operating system.
+	 * A function used to check if the given ZWJ emoji is supported in the operating system.
+	 *
+	 * Referenced for unit testing purposes.
+	 */
+	private static _isEmojiZwjSupported = isEmojiZwjSupported;
+
+	/**
+	 * A function used to get the emoji version supported by the operating system.
+	 *
+	 * Referenced for unit testing purposes.
+	 */
+	private static _getEmojiSupportedVersionByOs = getEmojiSupportedVersionByOs;
+
+	/**
+	 * A function used to determine if emoji has a zero width joiner.
 	 *
 	 * Referenced for unit testing purposes.
 	 */
-	private static _isEmojiSupported = isEmojiSupported;
+	private static _hasZwj = hasZwj;
 }
 
 /**
@@ -264,6 +291,26 @@ async function loadEmojiDatabase( emojiDatabaseUrl: string ): Promise<Array<Emoj
 	return result;
 }
 
+/**
+ * Checks the supported emoji version by the OS, by sampling some representatives from different emoji releases.
+ */
+function getEmojiSupportedVersionByOs() {
+	return Object.entries( EMOJI_SUPPORT_LEVEL ).reduce( ( supportedVersion, [ emoji, version ] ) => {
+		if ( isEmojiSupported( emoji ) && version > supportedVersion ) {
+			supportedVersion = version;
+		}
+
+		return supportedVersion;
+	}, 0 );
+}
+
+/**
+ * Check for ZWJ (zero width joiner) character.
+ */
+function hasZwj( emoji: string ) {
+	return emoji.includes( '\u200d' );
+}
+
 /**
  * Creates a div for emoji width testing purposes.
  */
@@ -296,14 +343,14 @@ function getNodeWidth( container: HTMLDivElement, node: string ): number {
 /**
  * Checks whether the emoji is supported in the operating system.
  */
-function isEmojiSupported( item: EmojiCdnResource, container: HTMLDivElement ): boolean {
+function isEmojiZwjSupported( item: EmojiCdnResource, container: HTMLDivElement ): boolean {
 	const emojiWidth = getNodeWidth( container, item.emoji );
 
 	// On Windows, some supported emoji are ~50% bigger than the baseline emoji, but what we really want to guard
 	// against are the ones that are 2x the size, because those are truly broken (person with red hair = person with
 	// floating red wig, black cat = cat with black square, polar bear = bear with snowflake, etc.)
 	// So here we set the threshold at 1.8 times the size of the baseline emoji.
-	return ( emojiWidth / 1.8 < BASELINE_EMOJI_WIDTH ) && ( emojiWidth >= BASELINE_EMOJI_WIDTH );
+	return emojiWidth < BASELINE_EMOJI_WIDTH * 1.8;
 }
 
 /**

packages/ckeditor5-emoji/src/utils/isemojisupported.ts

@@ -0,0 +1,77 @@
+/**
+ * @license Copyright (c) 2023, Koala Interactive SAS
+ * For licensing, see https://github.com/koala-interactive/is-emoji-supported/blob/master/LICENSE.md
+ */
+
+/**
+ * Checks if the two pixels parts are the same using canvas.
+ */
+export default function isEmojiSupported( unicode: string ): boolean {
+	const ctx = getCanvas();
+
+	/* istanbul ignore next -- @preserve */
+	if ( !ctx ) {
+		return false;
+	}
+
+	const CANVAS_HEIGHT = 25;
+	const CANVAS_WIDTH = 20;
+	const textSize = Math.floor( CANVAS_HEIGHT / 2 );
+
+	// Initialize canvas context.
+	ctx.font = textSize + 'px Arial, Sans-Serif';
+	ctx.textBaseline = 'top';
+	ctx.canvas.width = CANVAS_WIDTH * 2;
+	ctx.canvas.height = CANVAS_HEIGHT;
+
+	ctx.clearRect( 0, 0, CANVAS_WIDTH * 2, CANVAS_HEIGHT );
+
+	// Draw in red on the left.
+	ctx.fillStyle = '#FF0000';
+	ctx.fillText( unicode, 0, 22 );
+
+	// Draw in blue on right.
+	ctx.fillStyle = '#0000FF';
+	ctx.fillText( unicode, CANVAS_WIDTH, 22 );
+
+	const a = ctx.getImageData( 0, 0, CANVAS_WIDTH, CANVAS_HEIGHT ).data;
+	const count = a.length;
+	let i = 0;
+
+	// Search the first visible pixel.
+	for ( ; i < count && !a[ i + 3 ]; i += 4 ) ;
+
+	// No visible pixel.
+	/* istanbul ignore next -- @preserve */
+	if ( i >= count ) {
+		return false;
+	}
+
+	// Emoji has immutable color, so we check the color of the emoji in two different colors.
+	// the result show be the same.
+	const x = CANVAS_WIDTH + ( ( i / 4 ) % CANVAS_WIDTH );
+	const y = Math.floor( i / 4 / CANVAS_WIDTH );
+	const b = ctx.getImageData( x, y, 1, 1 ).data;
+
+	if ( a[ i ] !== b[ 0 ] || a[ i + 2 ] !== b[ 2 ]) {
+		return false;
+	}
+
+	// Some emojis are a contraction of different ones, so if it's not supported, it will show multiple characters.
+	/* istanbul ignore next -- @preserve */
+	if ( ctx.measureText( unicode ).width >= CANVAS_WIDTH ) {
+		return false;
+	}
+
+	// Supported.
+	return true;
+};
+
+function getCanvas(): CanvasRenderingContext2D | null {
+	try {
+		return document.createElement( 'canvas' ).getContext( '2d' );
+	} catch {
+		/* istanbul ignore next -- @preserve */
+		return null;
+	}
+}

packages/ckeditor5-emoji/tests/emojirepository.js

@@ -15,10 +15,12 @@ import EmojiRepository from '../src/emojirepository.js';
 describe( 'EmojiRepository', () => {
 	testUtils.createSinonSandbox();
 
-	let isEmojiSupportedStub, consoleStub, fetchStub;
+	let isEmojiSupportedStub, getEmojiSupportedVersionByOsStub, hasZwjStub, consoleStub, fetchStub;
 
 	beforeEach( () => {
-		isEmojiSupportedStub = testUtils.sinon.stub( EmojiRepository, '_isEmojiSupported' ).returns( true );
+		isEmojiSupportedStub = testUtils.sinon.stub( EmojiRepository, '_isEmojiZwjSupported' ).returns( true );
+		getEmojiSupportedVersionByOsStub = testUtils.sinon.stub( EmojiRepository, '_getEmojiSupportedVersionByOs' ).returns( 100 );
+		hasZwjStub = testUtils.sinon.stub( EmojiRepository, '_hasZwj' ).returns( false );
 
 		consoleStub = sinon.stub( console, 'warn' );
 		fetchStub = testUtils.sinon.stub( window, 'fetch' ).resolves( new Response( '[]' ) );
@@ -78,8 +80,8 @@ describe( 'EmojiRepository', () => {
 
 		it( 'should fetch the emoji database version 16', async () => {
 			const response = JSON.stringify( [
-				{ annotation: 'neutral face', group: 0 },
-				{ annotation: 'unamused face', group: 0 }
+				{ annotation: 'neutral face', group: 0, version: 16 },
+				{ annotation: 'unamused face', group: 0, version: 16 }
 			] );
 
 			fetchStubResolve( new Response( response ) );
@@ -140,29 +142,61 @@ describe( 'EmojiRepository', () => {
 			expect( hasGroup2 ).to.be.undefined;
 		} );
 
-		it( 'should filter out unsupported emojis from the fetched emoji database', async () => {
-			isEmojiSupportedStub.callsFake( item => item.annotation !== 'microscope' );
+		it( 'should filter out unsupported ZWJ emojis from the fetched emoji database', async () => {
+			// Neutral face is mocked to be the only non ZWJ icon.
+			hasZwjStub.callsFake( emoji => emoji !== '😐️' );
+
+			// Unamused face is the only supported ZWJ emoji.
+			isEmojiSupportedStub.callsFake( item => item.annotation === 'unamused face' );
 
 			const response = JSON.stringify( [
-				{ annotation: 'neutral face', group: 0 },
-				{ annotation: 'unamused face', group: 0 },
-				{ annotation: 'microscope', group: 7 }
+				{ emoji: '😐️', annotation: 'neutral face', group: 0, version: 16 },
+				{ emoji: '😒', annotation: 'unamused face', group: 0, version: 15 },
+				{ emoji: '🔬', annotation: 'microscope', group: 7, version: 15 }
+			] );
+
+			fetchStubResolve( new Response( response ) );
+
+			editor = await editorPromise;
+
+			const emojiRepositoryPlugin = editor.plugins.get( EmojiRepository );
+			const hasNeutralFaceEmoji = emojiRepositoryPlugin._database.find( item => item.annotation === 'neutral face' );
+			const hasMicroscopeEmoji = emojiRepositoryPlugin._database.find( item => item.annotation === 'microscope' );
+			const hasUnamusedEmoji = emojiRepositoryPlugin._database.find( item => item.annotation === 'unamused face' );
+
+			expect( hasNeutralFaceEmoji ).not.to.be.undefined; // Neutral face is not ZWJ, so it should not be filtered out.
+			expect( hasUnamusedEmoji ).not.to.be.undefined; // Unamused face is ZWJ, and it is supported, so it should not be filtered out.
+			expect( hasMicroscopeEmoji ).to.be.undefined; // Microscope is ZWJ, and it is not supported, so it should be filtered out
+		} );
+
+		it( 'should filter out emojis based on the version supported by the operating system', async () => {
+			getEmojiSupportedVersionByOsStub.reset();
+			getEmojiSupportedVersionByOsStub.returns( 15 );
+
+			const response = JSON.stringify( [
+				{ annotation: 'neutral face', group: 0, version: 16 },
+				{ annotation: 'unamused face', group: 0, version: 15 },
+				{ annotation: 'microscope', group: 7, version: 15 }
 			] );
 
 			fetchStubResolve( new Response( response ) );
 
 			editor = await editorPromise;
 
 			const emojiRepositoryPlugin = editor.plugins.get( EmojiRepository );
+			const hasNeutralFaceEmoji = emojiRepositoryPlugin._database.find( item => item.annotation === 'neutral face' );
 			const hasMicroscopeEmoji = emojiRepositoryPlugin._database.find( item => item.annotation === 'microscope' );
+			const hasUnamusedEmoji = emojiRepositoryPlugin._database.find( item => item.annotation === 'unamused face' );
 
-			expect( hasMicroscopeEmoji ).to.be.undefined;
+			expect( hasNeutralFaceEmoji ).to.be.undefined;
+			expect( hasMicroscopeEmoji ).not.to.be.undefined;
+			expect( hasUnamusedEmoji ).not.to.be.undefined;
 		} );
 
 		it( 'should set default skin tone for each emoji', async () => {
 			const response = JSON.stringify( [
-				{ annotation: 'neutral face', emoji: '😐️', group: 0 },
-				{ annotation: 'unamused face', emoji: '😒', group: 0 }
+				{ annotation: 'neutral face', emoji: '😐️', group: 0, version: 15 },
+				{ annotation: 'unamused face', emoji: '😒', group: 0, version: 15 }
 			] );
 
 			fetchStubResolve( new Response( response ) );
@@ -185,7 +219,7 @@ describe( 'EmojiRepository', () => {
 			const ninjaEmoji5 = '🥷🏿';
 
 			const response = JSON.stringify( [
-				{ annotation: 'ninja', emoji: ninjaEmoji0, group: 1, skins: [
+				{ annotation: 'ninja', emoji: ninjaEmoji0, group: 1, version: 15, skins: [
 					{ emoji: ninjaEmoji1, tone: 1 },
 					{ emoji: ninjaEmoji2, tone: 2 },
 					{ emoji: ninjaEmoji3, tone: 3 },
@@ -213,8 +247,8 @@ describe( 'EmojiRepository', () => {
 
 		it( 'should create Fuse.js instance with the emoji database', async () => {
 			const response = JSON.stringify( [
-				{ annotation: 'neutral face', group: 0 },
-				{ annotation: 'unamused face', group: 0 }
+				{ annotation: 'neutral face', group: 0, version: 15 },
+				{ annotation: 'unamused face', group: 0, version: 15 }
 			] );
 
 			fetchStubResolve( new Response( response ) );
@@ -274,8 +308,20 @@ describe( 'EmojiRepository', () => {
 
 		beforeEach( async () => {
 			const response = JSON.stringify( [
-				{ annotation: 'neutral face', emoticon: ':|', tags: [ 'awkward', 'blank', 'face', 'meh', 'whatever' ], group: 0 },
-				{ annotation: 'unamused face', emoticon: ':?', tags: [ 'bored', 'face', 'fine', 'ugh', 'whatever' ], group: 0 }
+				{
+					annotation: 'neutral face',
+					emoticon: ':|',
+					tags: [ 'awkward', 'blank', 'face', 'meh', 'whatever' ],
+					group: 0,
+					version: 15
+				},
+				{
+					annotation: 'unamused face',
+					emoticon: ':?',
+					tags: [ 'bored', 'face', 'fine', 'ugh', 'whatever' ],
+					group: 0,
+					version: 15
+				}
 			] );
 
 			fetchStub.resolves( new Response( response ) );
@@ -368,16 +414,16 @@ describe( 'EmojiRepository', () => {
 
 		beforeEach( async () => {
 			const response = JSON.stringify( [
-				{ annotation: 'neutral face', group: 0 },
-				{ annotation: 'ninja', group: 1 },
-				{ annotation: 'medium-dark skin tone', group: 2 },
-				{ annotation: 'lobster', group: 3 },
-				{ annotation: 'salt', group: 4 },
-				{ annotation: 'watch', group: 5 },
-				{ annotation: 'magic wand', group: 6 },
-				{ annotation: 'x-ray', group: 7 },
-				{ annotation: 'up-left arrow', group: 8 },
-				{ annotation: 'flag: Poland', group: 9 }
+				{ annotation: 'neutral face', group: 0, version: 15 },
+				{ annotation: 'ninja', group: 1, version: 15 },
+				{ annotation: 'medium-dark skin tone', group: 2, version: 15 },
+				{ annotation: 'lobster', group: 3, version: 15 },
+				{ annotation: 'salt', group: 4, version: 15 },
+				{ annotation: 'watch', group: 5, version: 15 },
+				{ annotation: 'magic wand', group: 6, version: 15 },
+				{ annotation: 'x-ray', group: 7, version: 15 },
+				{ annotation: 'up-left arrow', group: 8, version: 15 },
+				{ annotation: 'flag: Poland', group: 9, version: 15 }
 			] );
 
 			fetchStub.resolves( new Response( response ) );