TW-1492: Ads replacement rework for mobile browsing (#167)

* TW-1464 Prepare backend for new native ads

* TW-1464 Update API documentation

* TW-1464 Enable adding native ads in replace-only mode

* TW-1492 Add some properties for ads definitions

* TW-1492 Prepare the backend to additional fixtures for Mises

* TW-1492 Add an entrypoint for elements to hide or remove

Author: keshan3262

src/advertising/external-ads.ts

@@ -11,6 +11,11 @@ export const stylePropsNames = [
   'aspect-ratio',
   'background',
   'border',
+  'border-top',
+  'border-bottom',
+  'border-left',
+  'border-right',
+  'border-color',
   'border-radius',
   'bottom',
   'box-shadow',
@@ -49,6 +54,7 @@ export const stylePropsNames = [
   'min-inline-size',
   'min-width',
   'opacity',
+  'order',
   'overflow',
   'overflow-anchor',
   'overflow-wrap',
@@ -127,8 +133,10 @@ export interface PermanentAdPlacesRule extends ExtVersionConstraints {
   divWrapperStyle?: Record<StylePropName, string>;
   wrapperStyle?: Record<StylePropName, string>;
   elementToMeasureSelector?: string;
+  elementsToMeasureSelectors?: Record<'width' | 'height', string>;
   stylesOverrides?: AdStylesOverrides[];
   shouldHideOriginal?: boolean;
+  displayWidth?: string;
 }
 
 export interface AdProvidersByDomainRule extends ExtVersionConstraints {
@@ -138,7 +146,9 @@ export interface AdProvidersByDomainRule extends ExtVersionConstraints {
 
 export interface AdProviderSelectorsRule extends ExtVersionConstraints {
   selectors: string[];
+  negativeSelectors?: string[];
   parentDepth?: number;
+  enableForMises?: boolean;
 }
 
 export interface AdProviderForAllSitesRule extends ExtVersionConstraints {
@@ -149,13 +159,22 @@ export interface ReplaceAdsUrlsBlacklistEntry extends ExtVersionConstraints {
   regexes: string[];
 }
 
+export interface ElementsToHideOrRemoveEntry extends ExtVersionConstraints {
+  cssString: string;
+  parentDepth: number;
+  isMultiple: boolean;
+  urlRegexes: string[];
+  shouldHide: boolean;
+}
+
 const AD_PLACES_RULES_KEY = 'ad_places_rules';
 const AD_PROVIDERS_BY_SITES_KEY = 'ad_providers_by_sites';
 const AD_PROVIDERS_ALL_SITES_KEY = 'ad_providers_all_sites';
 const AD_PROVIDERS_LIST_KEY = 'ad_providers_list';
 const PERMANENT_AD_PLACES_RULES_KEY = 'permanent_ad_places_rules';
 const PERMANENT_NATIVE_AD_PLACES_RULES_KEY = 'permanent_native_ad_places_rules';
 const REPLACE_ADS_URLS_BLACKLIST_KEY = 'replace_ads_urls_blacklist';
+const ELEMENTS_TO_HIDE_OR_REMOVE_KEY = 'elements_to_hide_or_remove';
 
 export const adPlacesRulesMethods = objectStorageMethodsFactory<AdPlacesRule[]>(AD_PLACES_RULES_KEY, []);
 
@@ -181,6 +200,11 @@ export const replaceAdsUrlsBlacklistMethods = objectStorageMethodsFactory<Replac
   []
 );
 
+export const elementsToHideOrRemoveMethods = objectStorageMethodsFactory<ElementsToHideOrRemoveEntry[]>(
+  ELEMENTS_TO_HIDE_OR_REMOVE_KEY,
+  []
+);
+
 export const getAdProvidersForAllSites = async () => redisClient.smembers(AD_PROVIDERS_ALL_SITES_KEY);
 
 export const addAdProvidersForAllSites = async (providers: string[]) =>
@@ -191,5 +215,19 @@ export const removeAdProvidersForAllSites = async (providers: string[]) =>
 
 const FALLBACK_VERSION = '0.0.0';
 
-export const filterByVersion = <T extends ExtVersionConstraints>(rules: T[], version?: string) =>
-  rules.filter(({ extVersion }) => versionSatisfiesRange(version ?? FALLBACK_VERSION, extVersion));
+export function filterRules<T extends ExtVersionConstraints>(rules: T[], version: string | undefined): T[];
+export function filterRules<T extends ExtVersionConstraints & { enableForMises?: boolean }>(
+  rules: T[],
+  version: string | undefined,
+  isMisesBrowser: boolean
+): T[];
+export function filterRules<T extends ExtVersionConstraints & { enableForMises?: boolean }>(
+  rules: T[],
+  version: string | undefined,
+  isMisesBrowser = false
+) {
+  return rules.filter(
+    ({ extVersion, enableForMises = true }) =>
+      versionSatisfiesRange(version ?? FALLBACK_VERSION, extVersion) && (!isMisesBrowser || enableForMises)
+  );
+}

src/routers/slise-ad-rules/ad-places.ts

@@ -1,7 +1,7 @@
 import { Request, Router } from 'express';
 
 import {
-  filterByVersion,
+  filterRules,
   permanentNativeAdPlacesMethods,
   permanentAdPlacesMethods,
   adPlacesRulesMethods,
@@ -18,7 +18,7 @@ import {
 } from '../../utils/schemas';
 
 const transformAdPlaces = <T extends ExtVersionConstraints>(value: T[], req: Request) =>
-  filterByVersion(value, req.query.extVersion as string | undefined);
+  filterRules(value, req.query.extVersion as string | undefined);
 const transformAdPlacesDictionary = <T extends ExtVersionConstraints>(rules: Record<string, T[]>, req: Request) =>
   transformValues(rules, value => transformAdPlaces(value, req));
 
@@ -280,6 +280,18 @@ const transformAdPlacesDictionary = <T extends ExtVersionConstraints>(rules: Rec
  *             elementToMeasureSelector:
  *               type: string
  *               description: A selector of the element which should be measured to define banner size
+ *             elementsToMeasureSelectors:
+ *               type: object
+ *               required:
+ *                 - width
+ *                 - height
+ *               properties:
+ *                 width:
+ *                   type: string
+ *                   description: A selector of the element which should be measured to define banner width
+ *                 height:
+ *                   type: string
+ *                   description: A selector of the element which should be measured to define banner height
  *             stylesOverrides:
  *               type: array
  *               items:
@@ -288,6 +300,12 @@ const transformAdPlacesDictionary = <T extends ExtVersionConstraints>(rules: Rec
  *               type: boolean
  *               description: Whether original ads banners should be hidden but not removed
  *               default: false
+ *             displayWidth:
+ *               type: string
+ *               description: >
+ *                 A range of display widths in a semver-like format where the rule is applicable. Numbers can be only
+ *                 integers. If not specified, the rule is applicable for all display widths.
+ *               example: '>=1024 <1280'
  *           example:
  *             urlRegexes:
  *               - '^https://etherscan\.io/tx/'

src/routers/slise-ad-rules/elements-to-hide-or-remove.ts

@@ -0,0 +1,210 @@
+import { Router, Request } from 'express';
+
+import {
+  ElementsToHideOrRemoveEntry,
+  elementsToHideOrRemoveMethods,
+  filterRules
+} from '../../advertising/external-ads';
+import { addObjectStorageMethodsToRouter } from '../../utils/express-helpers';
+import { transformValues } from '../../utils/helpers';
+import { elementsToHideOrRemoveDictionarySchema, hostnamesListSchema } from '../../utils/schemas';
+
+export const elementsToHideOrRemoveRouter = Router();
+
+const transformElementsToHideOrRemoveRules = (value: ElementsToHideOrRemoveEntry[], req: Request) =>
+  filterRules(value, req.query.extVersion as string | undefined);
+const transformRulesDictionary = (value: Record<string, ElementsToHideOrRemoveEntry[]>, req: Request) =>
+  transformValues(value, rules => filterRules(rules, req.query.extVersion as string | undefined));
+
+/**
+ * @swagger
+ * tags:
+ *   name: Elements to hide or remove
+ * components:
+ *   schemas:
+ *     ElementsToHideOrRemoveEntry:
+ *       allOf:
+ *         - $ref: '#/components/schemas/ExtVersionConstraints'
+ *         - type: object
+ *           required:
+ *             - cssString
+ *             - parentDepth
+ *             - isMultiple
+ *             - urlRegexes
+ *             - shouldHide
+ *           properties:
+ *             cssString:
+ *               type: string
+ *             parentDepth:
+ *               type: number
+ *               min: 0
+ *               integer: true
+ *               description: >
+ *                 Indicates the depth of the parent element of the selected element
+ *             isMultiple:
+ *               type: boolean
+ *               description: Whether the selector should select multiple elements
+ *             urlRegexes:
+ *               type: array
+ *               items:
+ *                 type: string
+ *                 format: regex
+ *             shouldHide:
+ *               type: boolean
+ *     ElementsToHideOrRemoveDictionary:
+ *       type: object
+ *       additionalProperties:
+ *         type: array
+ *         items:
+ *           $ref: '#/components/schemas/ElementsToHideOrRemoveEntry'
+ *       example:
+ *         'm.economictimes.com':
+ *           - extVersion: '>=1.21.1'
+ *             cssString: '#iframeDisplay, #closeDisplay'
+ *             parentDepth: 0
+ *             isMultiple: true
+ *             urlRegexes:
+ *               - "^https://m\\.economictimes\\.com"
+ *             shouldHide: true
+ * /api/slise-ad-rules/elements-to-hide-or-remove/raw/all:
+ *   get:
+ *     summary: Get all rules for hiding or removing elements
+ *     tags:
+ *       - Elements to hide or remove
+ *     responses:
+ *       '200':
+ *         description: A dictionary of all rules
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/ElementsToHideOrRemoveDictionary'
+ *       '500':
+ *         $ref: '#/components/responses/ErrorResponse'
+ * /api/slise-ad-rules/elements-to-hide-or-remove/{domain}/raw:
+ *   get:
+ *     summary: Get rules for hiding or removing elements by domain
+ *     tags:
+ *       - Elements to hide or remove
+ *     parameters:
+ *       - name: domain
+ *         in: path
+ *         required: true
+ *         schema:
+ *           type: string
+ *           description: Domain name
+ *     responses:
+ *       '200':
+ *         description: An array of rules
+ *         content:
+ *           application/json:
+ *             schema:
+ *               type: array
+ *               items:
+ *                 $ref: '#/components/schemas/ElementsToHideOrRemoveEntry'
+ *       '500':
+ *         $ref: '#/components/responses/ErrorResponse'
+ * /api/slise-ad-rules/elements-to-hide-or-remove/{domain}:
+ *   get:
+ *     summary: Get rules for hiding or removing elements by domain and extension version
+ *     tags:
+ *       - Elements to hide or remove
+ *     parameters:
+ *       - name: domain
+ *         in: path
+ *         required: true
+ *         schema:
+ *           type: string
+ *           description: Domain name
+ *       - name: extVersion
+ *         in: query
+ *         schema:
+ *           type: string
+ *           default: '0.0.0'
+ *           description: Extension version
+ *     responses:
+ *       '200':
+ *         description: An array of rules
+ *         content:
+ *           application/json:
+ *             schema:
+ *               type: array
+ *               items:
+ *                 $ref: '#/components/schemas/ElementsToHideOrRemoveEntry'
+ *       '500':
+ *         $ref: '#/components/responses/ErrorResponse'
+ * /api/slise-ad-rules/elements-to-hide-or-remove:
+ *   get:
+ *     summary: Get all rules for hiding or removing elements filtered by extension version
+ *     tags:
+ *       - Elements to hide or remove
+ *     parameters:
+ *       - name: extVersion
+ *         in: query
+ *         schema:
+ *           type: string
+ *           default: '0.0.0'
+ *           description: Extension version
+ *     responses:
+ *       '200':
+ *         description: A dictionary of rules
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/ElementsToHideOrRemoveDictionary'
+ *       '500':
+ *         $ref: '#/components/responses/ErrorResponse'
+ *   post:
+ *     summary: Add rules for hiding or removing elements. If a rule already exists, it will be updated
+ *     tags:
+ *       - Elements to hide or remove
+ *     security:
+ *       - basicAuth: []
+ *     requestBody:
+ *       content:
+ *         application/json:
+ *           schema:
+ *            $ref: '#/components/schemas/ElementsToHideOrRemoveDictionary'
+ *     responses:
+ *       '200':
+ *         $ref: '#/components/responses/SuccessResponse'
+ *       '400':
+ *         $ref: '#/components/responses/ErrorResponse'
+ *       '401':
+ *         $ref: '#/components/responses/UnauthorizedError'
+ *       '500':
+ *         $ref: '#/components/responses/ErrorResponse'
+ *   delete:
+ *     summary: Delete rules for hiding or removing elements
+ *     tags:
+ *       - Elements to hide or remove
+ *     security:
+ *       - basicAuth: []
+ *     requestBody:
+ *       content:
+ *         application/json:
+ *           schema:
+ *             type: array
+ *             items:
+ *               type: string
+ *           example:
+ *             - 'm.economictimes.com'
+ *     responses:
+ *       '200':
+ *         $ref: '#/components/responses/SuccessResponse'
+ *       '400':
+ *         $ref: '#/components/responses/ErrorResponse'
+ *       '401':
+ *         $ref: '#/components/responses/UnauthorizedError'
+ *       '500':
+ *         $ref: '#/components/responses/ErrorResponse'
+ */
+addObjectStorageMethodsToRouter(elementsToHideOrRemoveRouter, {
+  path: '/',
+  methods: elementsToHideOrRemoveMethods,
+  keyName: 'domain',
+  objectValidationSchema: elementsToHideOrRemoveDictionarySchema,
+  keysArrayValidationSchema: hostnamesListSchema,
+  successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`,
+  objectTransformFn: transformRulesDictionary,
+  valueTransformFn: transformElementsToHideOrRemoveRules
+});

src/routers/slise-ad-rules/index.ts

@@ -1,6 +1,7 @@
 import { Router } from 'express';
 
 import { adPlacesRulesRouter } from './ad-places';
+import { elementsToHideOrRemoveRouter } from './elements-to-hide-or-remove';
 import { adProvidersRouter } from './providers';
 import { replaceUrlsBlacklistRouter } from './replace-urls-blacklist';
 
@@ -43,3 +44,4 @@ export const adRulesRouter = Router();
 adRulesRouter.use('/ad-places', adPlacesRulesRouter);
 adRulesRouter.use('/providers', adProvidersRouter);
 adRulesRouter.use('/replace-urls-blacklist', replaceUrlsBlacklistRouter);
+adRulesRouter.use('/elements-to-hide-or-remove', elementsToHideOrRemoveRouter);