Release TW Mobile v1.28 (#161)

Merge pull request #161 from madfish-solutions/development

Author: alex-tsx

package.json

@@ -53,6 +53,7 @@
     "@types/express": "^4.17.17",
     "@types/express-jwt": "^7.4.2",
     "@types/express-unless": "^2.0.1",
+    "@types/lodash": "^4.17.0",
     "@types/memoizee": "^0.4.5",
     "@types/node": "^18.14.6",
     "@types/pino": "^6.3.8",

src/advertising/external-ads.ts

@@ -84,7 +84,7 @@ interface AdStylesOverrides {
   style: Record<StylePropName, string>;
 }
 
-interface ExtVersionConstraints {
+export interface ExtVersionConstraints {
   extVersion: string;
 }
 
@@ -132,18 +132,24 @@ export interface AdProvidersByDomainRule extends ExtVersionConstraints {
 
 export interface AdProviderSelectorsRule extends ExtVersionConstraints {
   selectors: string[];
+  parentDepth?: number;
 }
 
 export interface AdProviderForAllSitesRule extends ExtVersionConstraints {
   providers: string[];
 }
 
+export interface ReplaceAdsUrlsBlacklistEntry extends ExtVersionConstraints {
+  regexes: string[];
+}
+
 const AD_PLACES_RULES_KEY = 'ad_places_rules';
 const AD_PROVIDERS_BY_SITES_KEY = 'ad_providers_by_sites';
 const AD_PROVIDERS_ALL_SITES_KEY = 'ad_providers_all_sites';
 const AD_PROVIDERS_LIST_KEY = 'ad_providers_list';
 const PERMANENT_AD_PLACES_RULES_KEY = 'permanent_ad_places_rules';
 const PERMANENT_NATIVE_AD_PLACES_RULES_KEY = 'permanent_native_ad_places_rules';
+const REPLACE_ADS_URLS_BLACKLIST_KEY = 'replace_ads_urls_blacklist';
 
 export const adPlacesRulesMethods = objectStorageMethodsFactory<AdPlacesRule[]>(AD_PLACES_RULES_KEY, []);
 
@@ -164,6 +170,11 @@ export const permanentNativeAdPlacesMethods = objectStorageMethodsFactory<Perman
   []
 );
 
+export const replaceAdsUrlsBlacklistMethods = objectStorageMethodsFactory<ReplaceAdsUrlsBlacklistEntry[]>(
+  REPLACE_ADS_URLS_BLACKLIST_KEY,
+  []
+);
+
 export const getAdProvidersForAllSites = async () => redisClient.smembers(AD_PROVIDERS_ALL_SITES_KEY);
 
 export const addAdProvidersForAllSites = async (providers: string[]) =>

src/getDAppsStats.ts

@@ -1,11 +1,11 @@
-import { dappList } from './utils/dapp-list-constants';
+import { DAPPS_LIST, IOS_DAPPS_LIST } from './utils/dapp-list-constants';
 import logger from './utils/logger';
 
-const getDAppsStats = async () => {
+const getDAppsStats = (forIOs: boolean) => {
   logger.info('Getting dApps list...');
 
   return {
-    dApps: dappList
+    dApps: forIOs ? IOS_DAPPS_LIST : DAPPS_LIST
   };
 };
 

src/index.ts

@@ -66,8 +66,6 @@ app.use(pinoHttp(PINO_LOGGER));
 app.use(cors());
 app.use(bodyParser.json());
 
-const dAppsProvider = new SingleQueryDataProvider(15 * 60 * 1000, getDAppsStats);
-
 const androidApp = firebaseAdmin.initializeApp(
   {
     projectId: 'templewallet-fa3b3'
@@ -162,7 +160,13 @@ app.post('/api/notifications', basicAuth, async (req, res) => {
   }
 });
 
-app.get('/api/dapps', makeProviderDataRequestHandler(dAppsProvider));
+app.get('/api/dapps', (req, res) => {
+  const platform = req.query.platform;
+
+  const data = getDAppsStats(platform === 'ios');
+
+  res.status(200).header('Cache-Control', 'public, max-age=300').send(data);
+});
 
 app.get('/api/abtest', (_, res) => {
   const data = getABData();

src/routers/slise-ad-rules/ad-places.ts

@@ -1,18 +1,27 @@
-import { Router } from 'express';
+import { Request, Router } from 'express';
 
 import {
   filterByVersion,
   permanentNativeAdPlacesMethods,
   permanentAdPlacesMethods,
-  adPlacesRulesMethods
+  adPlacesRulesMethods,
+  PermanentAdPlacesRule,
+  AdPlacesRule,
+  ExtVersionConstraints
 } from '../../advertising/external-ads';
 import { addObjectStorageMethodsToRouter } from '../../utils/express-helpers';
+import { transformValues } from '../../utils/helpers';
 import {
   hostnamesListSchema,
   permanentAdPlacesRulesDictionarySchema,
   adPlacesRulesDictionarySchema
 } from '../../utils/schemas';
 
+const transformAdPlaces = <T extends ExtVersionConstraints>(value: T[], req: Request) =>
+  filterByVersion(value, req.query.extVersion as string | undefined);
+const transformAdPlacesDictionary = <T extends ExtVersionConstraints>(rules: Record<string, T[]>, req: Request) =>
+  transformValues(rules, value => transformAdPlaces(value, req));
+
 /**
  * @swagger
  * tags:
@@ -68,13 +77,14 @@ import {
  *             type: string
  *     ExtVersionConstraints:
  *       type: object
+ *       required:
+ *         - extVersion
  *       properties:
  *         extVersion:
  *           type: string
  *           description: >
  *             A range of versions where the rule is applicable. If not specified, the rule is applicable
  *             for all versions. See the [ranges format](https://www.npmjs.com/package/semver#ranges)
- *           default: '*'
  *     AdPlacesRule:
  *       allOf:
  *         - $ref: '#/components/schemas/ExtVersionConstraints'
@@ -420,14 +430,15 @@ export const adPlacesRulesRouter = Router();
  *       '500':
  *         $ref: '#/components/responses/ErrorResponse'
  */
-addObjectStorageMethodsToRouter(adPlacesRulesRouter, {
+addObjectStorageMethodsToRouter<PermanentAdPlacesRule[]>(adPlacesRulesRouter, {
   path: '/permanent-native',
   methods: permanentNativeAdPlacesMethods,
   keyName: 'domain',
   objectValidationSchema: permanentAdPlacesRulesDictionarySchema,
   keysArrayValidationSchema: hostnamesListSchema,
   successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`,
-  transformGotValueFn: (value, req) => filterByVersion(value, req.query.extVersion as string | undefined)
+  valueTransformFn: transformAdPlaces,
+  objectTransformFn: transformAdPlacesDictionary
 });
 
 /**
@@ -569,14 +580,15 @@ addObjectStorageMethodsToRouter(adPlacesRulesRouter, {
  *       '500':
  *         $ref: '#/components/responses/ErrorResponse'
  */
-addObjectStorageMethodsToRouter(adPlacesRulesRouter, {
+addObjectStorageMethodsToRouter<PermanentAdPlacesRule[]>(adPlacesRulesRouter, {
   path: '/permanent',
   methods: permanentAdPlacesMethods,
   keyName: 'domain',
   objectValidationSchema: permanentAdPlacesRulesDictionarySchema,
   keysArrayValidationSchema: hostnamesListSchema,
   successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`,
-  transformGotValueFn: (value, req) => filterByVersion(value, req.query.extVersion as string | undefined)
+  valueTransformFn: transformAdPlaces,
+  objectTransformFn: transformAdPlacesDictionary
 });
 
 /**
@@ -718,12 +730,13 @@ addObjectStorageMethodsToRouter(adPlacesRulesRouter, {
  *       '500':
  *         $ref: '#/components/responses/ErrorResponse'
  */
-addObjectStorageMethodsToRouter(adPlacesRulesRouter, {
+addObjectStorageMethodsToRouter<AdPlacesRule[]>(adPlacesRulesRouter, {
   path: '/',
   methods: adPlacesRulesMethods,
   keyName: 'domain',
   objectValidationSchema: adPlacesRulesDictionarySchema,
   keysArrayValidationSchema: hostnamesListSchema,
   successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`,
-  transformGotValueFn: (value, req) => filterByVersion(value, req.query.extVersion as string | undefined)
+  valueTransformFn: transformAdPlaces,
+  objectTransformFn: transformAdPlacesDictionary
 });

src/routers/slise-ad-rules/index.ts

@@ -2,6 +2,7 @@ import { Router } from 'express';
 
 import { adPlacesRulesRouter } from './ad-places';
 import { adProvidersRouter } from './providers';
+import { replaceUrlsBlacklistRouter } from './replace-urls-blacklist';
 
 /**
  * @swagger
@@ -41,3 +42,4 @@ export const adRulesRouter = Router();
 
 adRulesRouter.use('/ad-places', adPlacesRulesRouter);
 adRulesRouter.use('/providers', adProvidersRouter);
+adRulesRouter.use('/replace-urls-blacklist', replaceUrlsBlacklistRouter);

src/routers/slise-ad-rules/providers.ts

@@ -1,4 +1,5 @@
-import { Router } from 'express';
+import { Request, Router } from 'express';
+import { identity } from 'lodash';
 
 import {
   addAdProvidersForAllSites,
@@ -7,12 +8,14 @@ import {
   adProvidersMethods,
   adProvidersByDomainRulesMethods,
   AdProviderSelectorsRule,
-  filterByVersion
+  filterByVersion,
+  AdProvidersByDomainRule
 } from '../../advertising/external-ads';
 import { basicAuth } from '../../middlewares/basic-auth.middleware';
 import { addObjectStorageMethodsToRouter, withBodyValidation, withExceptionHandler } from '../../utils/express-helpers';
+import { isDefined, transformValues } from '../../utils/helpers';
 import {
-  adTypesListSchema,
+  nonEmptyStringsListSchema,
   hostnamesListSchema,
   adProvidersByDomainsRulesDictionarySchema,
   adProvidersDictionarySchema
@@ -69,17 +72,36 @@ import {
  *               type: array
  *               items:
  *                 type: string
+ *             parentDepth:
+ *               type: integer
+ *               minimum: 0
+ *               default: 0
+ *     AdByProviderSelector:
+ *       oneOf:
+ *         - type: string
+ *         - type: object
+ *           required:
+ *             - selector
+ *             - parentDepth
+ *           properties:
+ *             selector:
+ *               type: string
+ *             parentDepth:
+ *               type: integer
  *     AdProvidersDictionary:
  *       type: object
  *       additionalProperties:
  *         type: array
  *         items:
- *           type: string
+ *           $ref: '#/components/schemas/AdByProviderSelector'
  *       example:
  *         google:
  *           - '#Ads_google_bottom_wide'
  *           - '.GoogleAdInfo'
  *           - 'a[href^="https://googleads.g.doubleclick.net/pcs/click"]'
+ *         persona:
+ *           - selector: "a.persona-product"
+ *             parentDepth: 1
  *     AdProvidersInputsDictionary:
  *       type: object
  *       additionalProperties:
@@ -178,7 +200,7 @@ adProvidersRouter
   .post(
     basicAuth,
     withExceptionHandler(
-      withBodyValidation(adTypesListSchema, async (req, res) => {
+      withBodyValidation(nonEmptyStringsListSchema, async (req, res) => {
         const providersAddedCount = await addAdProvidersForAllSites(req.body);
 
         res.status(200).send({ message: `${providersAddedCount} providers have been added` });
@@ -188,7 +210,7 @@ adProvidersRouter
   .delete(
     basicAuth,
     withExceptionHandler(
-      withBodyValidation(adTypesListSchema, async (req, res) => {
+      withBodyValidation(nonEmptyStringsListSchema, async (req, res) => {
         const providersRemovedCount = await removeAdProvidersForAllSites(req.body);
 
         res.status(200).send({ message: `${providersRemovedCount} providers have been removed` });
@@ -287,13 +309,15 @@ adProvidersRouter
  *       '500':
  *         $ref: '#/components/responses/ErrorResponse'
  */
-addObjectStorageMethodsToRouter(adProvidersRouter, {
+addObjectStorageMethodsToRouter<AdProvidersByDomainRule[]>(adProvidersRouter, {
   path: '/by-sites',
   methods: adProvidersByDomainRulesMethods,
   keyName: 'domain',
   objectValidationSchema: adProvidersByDomainsRulesDictionarySchema,
   keysArrayValidationSchema: hostnamesListSchema,
-  successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`
+  successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`,
+  valueTransformFn: identity,
+  objectTransformFn: identity
 });
 
 /**
@@ -361,11 +385,13 @@ addObjectStorageMethodsToRouter(adProvidersRouter, {
  *             schema:
  *               type: array
  *               items:
- *                 type: string
+ *                 $ref: '#/components/schemas/AdByProviderSelector'
  *               example:
  *                 - '#Ads_google_bottom_wide'
  *                 - '.GoogleAdInfo'
  *                 - 'a[href^="https://googleads.g.doubleclick.net/pcs/click"]'
+ *                 - selector: "a.persona-product"
+ *                   parentDepth: 1
  *       '500':
  *         $ref: '#/components/responses/ErrorResponse'
  * /api/slise-ad-rules/providers:
@@ -437,19 +463,30 @@ addObjectStorageMethodsToRouter(adProvidersRouter, {
  *       '500':
  *         $ref: '#/components/responses/ErrorResponse'
  */
-addObjectStorageMethodsToRouter<AdProviderSelectorsRule[], string[]>(adProvidersRouter, {
+const transformAdProviderSelectorsRules = (rules: AdProviderSelectorsRule[], req: Request) =>
+  Array.from(
+    new Set(
+      filterByVersion(rules, req.query.extVersion as string | undefined)
+        .map(({ selectors, parentDepth }) =>
+          isDefined(parentDepth) && parentDepth > 0 ? { selector: selectors.join(', '), parentDepth } : selectors
+        )
+        .flat()
+    )
+  );
+
+type AdByProviderSelector = string | { selector: string; parentDepth: number };
+
+addObjectStorageMethodsToRouter<
+  AdProviderSelectorsRule[],
+  Record<string, AdByProviderSelector[]>,
+  AdByProviderSelector[]
+>(adProvidersRouter, {
   path: '/',
   methods: adProvidersMethods,
   keyName: 'providerId',
   objectValidationSchema: adProvidersDictionarySchema,
-  keysArrayValidationSchema: adTypesListSchema,
+  keysArrayValidationSchema: nonEmptyStringsListSchema,
   successfulRemovalMessage: entriesCount => `${entriesCount} providers have been removed`,
-  transformGotValueFn: (rules, req) =>
-    Array.from(
-      new Set(
-        filterByVersion(rules, req.query.extVersion as string | undefined)
-          .map(({ selectors }) => selectors)
-          .flat()
-      )
-    )
+  valueTransformFn: transformAdProviderSelectorsRules,
+  objectTransformFn: (rules, req) => transformValues(rules, value => transformAdProviderSelectorsRules(value, req))
 });