feat(backend): vendor detection groundwork & data-request integration

assets/data/datarequests_companies.json

@@ -0,0 +1,146 @@
+{
+  "entities": {
+    "amazon": {
+      "id": "amazon",
+      "name": "Amazon",
+      "vendor_ids": [
+        "amazon_ses",
+        "amazon_aws"
+      ],
+      "entries": {
+        "amazon-fr": {
+          "slug": "amazon-fr",
+          "name": "Amazon Europe Core SARL",
+          "type": "controller",
+          "country": "FR",
+          "domains": [
+            "amazon.fr"
+          ],
+          "datarequests_url": "https://www.datarequests.org/company/amazon-fr/",
+          "gdpr": {
+            "email": "[email protected]"
+          }
+        },
+        "amazon-es": {
+          "slug": "amazon-es",
+          "name": "Amazon Europe Core SARL",
+          "type": "controller",
+          "country": "ES",
+          "domains": [
+            "amazon.es"
+          ],
+          "datarequests_url": "https://www.datarequests.org/company/amazon-es/",
+          "gdpr": {
+            "email": "[email protected]"
+          }
+        },
+        "amazon-de": {
+          "slug": "amazon-de",
+          "name": "Amazon EU S.à r.l.",
+          "type": "controller",
+          "country": "DE",
+          "domains": [
+            "amazon.de"
+          ],
+          "datarequests_url": "https://www.datarequests.org/company/amazon-de/"
+        },
+        "amazon-aws": {
+          "slug": "amazon-aws",
+          "name": "Amazon Web Services EMEA SARL",
+          "type": "processor",
+          "domains": [
+            "amazonaws.com"
+          ],
+          "datarequests_url": "https://www.datarequests.org/company/amazon-aws/"
+        }
+      }
+    },
+    "salesforce": {
+      "id": "salesforce",
+      "name": "Salesforce",
+      "vendor_ids": [
+        "salesforce"
+      ],
+      "entries": {
+        "salesforce": {
+          "slug": "salesforce",
+          "name": "Salesforce",
+          "type": "processor",
+          "domains": [
+            "salesforce.com",
+            "pardot.com",
+            "exacttarget.com"
+          ],
+          "datarequests_url": "https://www.datarequests.org/company/salesforce/"
+        }
+      }
+    },
+    "microsoft": {
+      "id": "microsoft",
+      "name": "Microsoft",
+      "vendor_ids": [
+        "microsoft_365"
+      ],
+      "entries": {
+        "microsoft": {
+          "slug": "microsoft",
+          "name": "Microsoft Corporation",
+          "type": "controller",
+          "domains": [
+            "microsoft.com",
+            "outlook.com",
+            "hotmail.com",
+            "live.com",
+            "office.com"
+          ],
+          "datarequests_url": "https://www.datarequests.org/company/microsoft/"
+        }
+      }
+    },
+    "coinmarketcap": {
+      "id": "coinmarketcap",
+      "name": "CoinMarketCap",
+      "entries": {
+        "coinmarketcap": {
+          "slug": "coinmarketcap",
+          "name": "CoinMarketCap",
+          "type": "controller",
+          "domains": [
+            "coinmarketcap.com"
+          ],
+          "datarequests_url": "https://www.datarequests.org/company/coinmarketcap/"
+        }
+      }
+    },
+    "123greetings": {
+      "id": "123greetings",
+      "name": "123Greetings.com",
+      "entries": {
+        "123greetings": {
+          "slug": "123greetings",
+          "name": "123Greetings.com",
+          "type": "controller",
+          "domains": [
+            "123greetings.com"
+          ],
+          "datarequests_url": "https://www.datarequests.org/company/123greetings/"
+        }
+      }
+    },
+    "numberly": {
+      "id": "numberly",
+      "name": "Numberly",
+      "entries": {
+        "numberly": {
+          "slug": "numberly",
+          "name": "Numberly",
+          "type": "controller",
+          "domains": [
+            "numberly.com"
+          ],
+          "datarequests_url": "https://www.datarequests.org/company/numberly/"
+        }
+      }
+    }
+  }
+}

assets/data/datarequests_domains.json

@@ -0,0 +1,27 @@
+{
+    "en": {
+      "base_url": "https://www.datarequests.org"
+    },
+    "fr": {
+      "base_url": "https://www.demandetesdonnees.fr"
+    },
+    "de": {
+      "base_url": "https://www.datenanfragen.de"
+    },
+    "nl": {
+      "base_url": "https://www.gegevensaanvragen.nl"
+    },
+    "es": {
+      "base_url": "https://www.solicituddedatos.es"
+    },
+    "pt": {
+      "base_url": "https://www.pedidodedados.org"
+    },
+    "hr": {
+      "base_url": "https://www.osobnipodaci.org"
+    },
+    "cs": {
+      "base_url": "https://www.zadostioudaje.org"
+    }
+  }
+

assets/data/vendor_senders.json

@@ -0,0 +1,160 @@
+{
+  "confidence_rules": {
+    "high": [
+      "dkim_domain",
+      "header_name",
+      "header_prefix"
+    ],
+    "medium": [
+      "return_path_domain",
+      "received_domain"
+    ],
+    "low": [
+      "from_domain",
+      "reply_to_domain"
+    ]
+  },
+  "vendors": [
+    {
+      "vendor_id": "salesforce",
+      "name": "Salesforce",
+      "platform_domains": [
+        "salesforce.com",
+        "salesforceiq.com",
+        "exacttarget.com",
+        "pardot.com"
+      ],
+      "dkim_domains": [
+        "salesforce.com",
+        "salesforceiq.com",
+        "exacttarget.com",
+        "pardot.com"
+      ],
+      "return_path_domains": [
+        "salesforce.com",
+        "exacttarget.com",
+        "pardot.com"
+      ],
+      "received_domains": [
+        "salesforce.com",
+        "exacttarget.com",
+        "pardot.com"
+      ],
+      "header_names": [
+        "x-sfdc-user",
+        "x-sfdc-lk",
+        "x-exacttarget-messageid"
+      ],
+      "header_prefixes": [
+        "x-sfdc-"
+      ]
+    },
+    {
+      "vendor_id": "amazon_ses",
+      "name": "Amazon SES",
+      "platform_domains": [
+        "amazonses.com"
+      ],
+      "dkim_domains": [
+        "amazonses.com"
+      ],
+      "return_path_domains": [
+        "amazonses.com"
+      ],
+      "received_domains": [
+        "amazonses.com"
+      ],
+      "header_names": [
+        "x-ses-outgoing"
+      ],
+      "header_prefixes": [
+        "x-ses-"
+      ]
+    },
+    {
+      "vendor_id": "sendgrid",
+      "name": "SendGrid",
+      "platform_domains": [
+        "sendgrid.net"
+      ],
+      "dkim_domains": [
+        "sendgrid.net"
+      ],
+      "return_path_domains": [
+        "sendgrid.net"
+      ],
+      "received_domains": [
+        "sendgrid.net"
+      ],
+      "header_names": [
+        "x-sg-eid",
+        "x-sg-id"
+      ],
+      "header_prefixes": [
+        "x-sg-"
+      ]
+    },
+    {
+      "vendor_id": "mailchimp",
+      "name": "Mailchimp",
+      "platform_domains": [
+        "mailchimpapp.net",
+        "mcsv.net"
+      ],
+      "dkim_domains": [
+        "mailchimpapp.net",
+        "mcsv.net"
+      ],
+      "return_path_domains": [
+        "mailchimpapp.net",
+        "mcsv.net"
+      ],
+      "received_domains": [
+        "mailchimpapp.net",
+        "mcsv.net"
+      ],
+      "header_names": [
+        "x-mailchimp-id"
+      ],
+      "header_prefixes": [
+        "x-mailchimp-"
+      ]
+    },
+    {
+      "vendor_id": "microsoft_365",
+      "name": "Microsoft Dynamics / Microsoft 365",
+      "platform_domains": [
+        "onmicrosoft.com",
+        "office365.com",
+        "outlook.com",
+        "protection.outlook.com",
+        "mail.protection.outlook.com",
+        "dynamics.com"
+      ],
+      "dkim_domains": [
+        "onmicrosoft.com",
+        "office365.com",
+        "outlook.com"
+      ],
+      "return_path_domains": [
+        "onmicrosoft.com",
+        "office365.com",
+        "outlook.com"
+      ],
+      "received_domains": [
+        "protection.outlook.com",
+        "mail.protection.outlook.com",
+        "outlook.com",
+        "office365.com"
+      ],
+      "header_names": [
+        "x-ms-exchange-organization-messageid",
+        "x-ms-exchange-organization-authsource",
+        "x-ms-exchange-organization-authas"
+      ],
+      "header_prefixes": [
+        "x-ms-"
+      ]
+    }
+  ]
+}

config/app.php

@@ -640,7 +640,7 @@
     | Handles page layout, login/logout, and the default settings pages. This set
     | is required.
     */
-    'modules' => explode(',', env('CYPHT_MODULES','core,contacts,local_contacts,feeds,imap,smtp,account,idle_timer,calendar,themes,nux,developer,history,saved_searches,advanced_search,highlights,profiles,inline_message,imap_folders,keyboard_shortcuts,tags')),
+    'modules' => explode(',', env('CYPHT_MODULES','core,contacts,local_contacts,feeds,imap,smtp,account,idle_timer,calendar,themes,nux,developer,history,saved_searches,advanced_search,highlights,profiles,inline_message,imap_folders,keyboard_shortcuts,tags,vendor_detection')),
     // 'modules' => [
     //     /*
     //     |  ----
@@ -1357,6 +1357,13 @@
     */
     'default_setting_enable_sieve_filter' => env('DEFAULT_SETTING_ENABLE_SIEVE_FILTER', false),
 
+    /*
+    |
+    | Enable platform/vendor blocking in sieve block sender
+    | Defaults to false
+    */
+    'enable_platform_blocking' => env('ENABLE_PLATFORM_BLOCKING', true),
+
     /*
     | Fancy Login page
     | Use this setting switch between the legacy login page and the fancy one