poweremail_mailbox.py

# -*- coding: utf-8 -*-
from osv import osv
from tools.translate import _
from tools import flatten
from talon import quotations
from datetime import datetime
from qreu.address import Address
from html2text import html2text
from lxml import html as lxml_html
import email as email_parser
import re

from .markdown_utils import normalize_markdown_image_descriptions

try:
    from urllib.parse import unquote
except ImportError:
    from urllib import unquote

import qreu


CASE_ID_RE = re.compile(r"<.*tinycrm-(\d+)@.*>", re.UNICODE)
MARKDOWN_EMAIL_AUTOLINK_RE = re.compile(
    r'<(mailto:)?([A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,})>',
    re.IGNORECASE | re.UNICODE
)

try:
    basestring
except NameError:
    basestring = str


def get_cases_ids_from_references(references):
    return list({
        int(x) for x in flatten([CASE_ID_RE.findall(ref) for ref in references])
    })


def _normalize_attachment_ref(value):
    if not value:
        return False
    if not isinstance(value, basestring):
        value = str(value)
    value = value.strip()
    if value.lower().startswith('cid:'):
        value = value[4:]
    value = unquote(value)
    return value.strip('<>')


def _attachment_ids_from_browse(attachments):
    return [attachment.id for attachment in attachments]


def _build_inline_attachment_map(raw_email, attachments):
    attachment_ids = _attachment_ids_from_browse(attachments)
    if not raw_email or not attachment_ids:
        return {}

    mail_message = email_parser.message_from_string(raw_email)
    attachment_map = {}
    part_index = 0
    for part in mail_message.walk():
        filename = part.get_filename()
        if not filename:
            continue
        if part_index >= len(attachment_ids):
            break
        attachment_id = attachment_ids[part_index]
        for value in (
                part.get('Content-ID'),
                part.get('Content-Location'),
                filename):
            key = _normalize_attachment_ref(value)
            if key:
                attachment_map[key] = attachment_id
        part_index += 1
    return attachment_map


def _replace_inline_image_sources(html_body, attachment_map):
    if not html_body or not attachment_map:
        return html_body
    try:
        fragment = lxml_html.fragment_fromstring(
            html_body, create_parent='div'
        )
    except Exception:
        return html_body

    changed = False
    for image in fragment.iter('img'):
        src = _normalize_attachment_ref(image.get('src'))
        if src in attachment_map:
            image.set('src', 'attachment://{0}'.format(attachment_map[src]))
            changed = True

    if not changed:
        return html_body
    parts = [fragment.text or '']
    parts.extend([
        lxml_html.tostring(child, encoding='unicode', method='html')
        for child in fragment
    ])
    return ''.join(parts)


def _escape_mdx_email_autolinks(markdown_body):
    if not markdown_body:
        return markdown_body
    return MARKDOWN_EMAIL_AUTOLINK_RE.sub(r'\2', markdown_body)


class PoweremailMailboxCRM(osv.osv):
    """Overwriting create.
    """
    _name = 'poweremail.mailbox'
    _inherit = 'poweremail.mailbox'


    def crm_udpate_partner_address(self, cursor, uid, address_id, email):
        address_obj = self.pool.get('res.partner.address')
        partner_obj = self.pool.get('res.partner')
        domain = email.split('@')[-1].strip()
        partner_id = partner_obj.search(
            cursor, uid, [('domain', '=', domain)]
        )
        if partner_id:
            address_obj.write(
                cursor, uid, address_id, {'partner_id': partner_id[0]}
            )


    def get_partner_address_from_email(self, cursor, uid, email_address):
        """
        Gets or Creates the 'res.partner.address' from a poweremail_mailbox from
        :param cursor:      OpenERP Cursor
        :param uid:         OpenERP User ID
        :type uid:          int
        :param email:       Email address
        :type email:        str
        :return:            Res.Partner.Address (browsed)
        :rtype:             osv.osv
        """
        address_obj = self.pool.get('res.partner.address')
        email = Address.parse(email_address)
        address_id = address_obj.search(cursor, uid, [
            ('email', '=ilike', email.address)
        ])
        if address_id:
            return address_id[0]
        else:
            # If not found: create partner address
            address_id = address_obj.create(cursor, uid, {
                'name': email.display_name,
                'email': email.address,
            })
            self.crm_udpate_partner_address(
                cursor, uid, address_id, email.address
            )
        return address_id

    def get_partner_address(self, cursor, uid, p_mail_id):
        """
        Gets or Creates the 'res.partner.address' from a poweremail_mailbox
        :param cursor:      OpenERP Cursor
        :param uid:         OpenERP User ID
        :type uid:          int
        :param p_mail_id:   PowerEmail Mailbox object ID
        :type p_mail_id:    int
        :return:            Res.Partner.Address (browsed)
        :rtype:             osv.osv
        """
        address_obj = self.pool.get('res.partner.address')
        p_mail = self.pool.get('poweremail.mailbox').read(
            cursor, uid, p_mail_id, ['pem_mail_orig', 'pem_from']
        )
        mail = qreu.Email.parse(p_mail['pem_mail_orig'])
        address_id = self.get_partner_address_from_email(cursor, uid,
                                                         mail.from_.display)
        return address_obj.browse(cursor, uid, address_id) or False

    def create_crm_case(self, cursor, uid, p_mail_id, section_id,
                        body_text=False, context=None):
        """
        Creates a CRM.case object from a poweremail.mailbox object
        :param cursor:      OpenERP Cursor
        :param uid:         OpenERP User ID
        :type uid:          int
        :param p_mail_id:   PowerEmail Mailbox object
        :type p_mail_id:    int
        :param section_id:  CRM.case.section that the new case must belong
        :type section_id:   int
        :param body_text:   Mail body parsed, used in the CRM.case.description
        :type body_text:    str
        :param context:     OpenERP Context, passed to the create method
        :type context:      dict
        :return:            New crm.case object (browsed)
        :rtype:             osv.osv
        """
        if context is None:
            context = {}
        case_obj = self.pool.get('crm.case')
        section_obj = self.pool.get('crm.case.section')
        poweremail_obj = self.pool.get('poweremail.mailbox')

        p_mail = poweremail_obj.read(cursor, uid, p_mail_id, [
            'conversation_id', 'pem_subject', 'pem_from', 'pem_cc'
        ])
        section = section_obj.read(cursor, uid, section_id, ['user_id'])

        case_vals = {
            'conversation_id': p_mail['conversation_id'][0],
            'name': p_mail['pem_subject'],
            'section_id': section['id'],
            'description': body_text,
            'email_from': p_mail['pem_from'],
            'email_cc': p_mail['pem_cc'],
            'user_id': section['user_id'][0] if section['user_id'] else False,
        }
        address = self.get_partner_address(
            cursor, uid, p_mail['id']
        )
        if address:
            case_vals.update({
                'partner_address_id': address.id,
                'partner_id': address.partner_id.id
            })
        return case_obj.create(cursor, uid, case_vals, context)

    def update_case_from_mail(
            self, cursor, uid, p_mail_id, case_id, email, context=None):
        """
        Update the case with the data from the poweremail and the email

        1. Description (History)
        2. Log (History)
        3. Addresses (Watchers CC)

        :param cursor:      OpenERP Cursor
        :param uid:         Res.User ID
        :param p_mail_id:   Poweremail.Mailbox ID
        :param case_id:     Crm.Case ID
        :param email:       Qreu.Email instance (related to the mailbox ID)
        :param context:     OpenERP Context
        :return:
        """
        if context is None:
            context = {}
        if isinstance(p_mail_id, (list, tuple)):
            p_mail_id = p_mail_id[0]
        if isinstance(case_id, (list, tuple)):
            case_id = case_id[0]
        p_mail = self.browse(cursor, uid, p_mail_id, context=context)
        case_obj = self.pool.get('crm.case')
        section_obj = self.pool.get('crm.case.section')

        case = case_obj.read(
            cursor, uid, case_id, ['description', 'state']
        )

        # 1.- Description
        old_descr = case['description']
        if old_descr:
            case_obj._history(
                cursor, uid,
                case_obj.browse(cursor, uid, [case_id]),
                _('Reply'), history=True,
                email=email.from_.address
            )
        body_text = quotations.extract_from_plain(p_mail.pem_body_text)
        case_obj.write(cursor, uid, case_id, {
            'description': body_text
        })

        # 2.- Logs
        case_obj._history(
            cursor, uid, case_obj.browse(cursor, uid, [case_id]),
            _('Reply'), history=True, email=email.from_.address
        )
        # 2.5 - If pending or done set to open again
        if case['state'] in ('pending', 'done'):
            case_obj.case_reopen_and_notification(
                cursor, uid, [case_id]
            )
        # 3.- Emails from CC, TO and FROM
        case_data = case_obj.read(cursor, uid, case_id, ['section_id'])
        reply_to = section_obj.read(
            cursor, uid, case_data['section_id'][0], ['reply_to']
        )['reply_to']
        addrs_ids = [
            self.get_partner_address_from_email(cursor, uid, addr)
            for addr in list(set(email.cc + email.to + [email.from_.display]))
            if reply_to not in addr and addr not in reply_to
        ]
        case_obj.add_to_watchers(
            cursor, uid,  case_id, address_ids=addrs_ids
        )

    def _email_body_as_markdown(self, p_mail, mail):
        html_body = p_mail.pem_body_html or mail.body_parts.get('html')
        if html_body:
            attachment_map = _build_inline_attachment_map(
                p_mail.pem_mail_orig, p_mail.pem_attachments_ids
            )
            html_body = _replace_inline_image_sources(
                html_body, attachment_map
            )
            markdown_body = normalize_markdown_image_descriptions(
                html2text(html_body).strip()
            )
            return _escape_mdx_email_autolinks(markdown_body)
        return p_mail.pem_body_text or mail.body_parts.get('plain') or ''

    def _markdown_inline_images_enabled(self, cursor, uid):
        conf_obj = self.pool.get('res.config')
        return bool(int(conf_obj.get(
            cursor, uid, 'crm_poweremail_markdown_inline_images', 0
        )))

    def forward_case_response(
            self, cursor, uid, pmail_id, case, email, context=None):
        """
        Make a forwarding poweremail.mailbox with all the recipients from the
        case, excluding those in the shared email
        :param cursor:      OpenERP Cursor
        :param uid:         OpenERP User ID
        :param pmail_id:    Poweremail.Mailbox ID
        :param case:        Browse Record
        :param email:       Qreu.Email instance
        :param context:     OpenERP Context
        :return:
        """
        case_obj = self.pool.get('crm.case')
        mailbox_obj = self.pool.get('poweremail.mailbox')
        email_from = case.section_id.reply_to
        email_to = case_obj.filter_mails(
            (
                case.get_cc_emails() +
                [case.user_id.address_id.email, case.partner_address_id.email]
            ),
            email.from_.address,
            case,
            todel_emails=email.recipients.addresses+[case.section_id.reply_to]
        )
        email_bcc = case_obj.filter_mails(
            case.get_bcc_emails(),
            email.from_.address,
            case,
            todel_emails=(
                email.recipients.addresses +
                [case.section_id.reply_to] +
                email_to
            )
        )
        if email_to:
            p_mail = self.browse(cursor, uid, pmail_id, context=context)
            vals_forward = {
                'pem_to': email_to[0],
                'pem_from': email_from,
                'pem_cc': ','.join(email_to[1:]) if len(email_to[1:]) else '',
                'pem_bcc': ','.join(email_bcc) if email_bcc else '',
                'pem_subject': p_mail.pem_subject,
                'pem_body_text': p_mail.pem_body_text,
                'pem_body_html': p_mail.pem_body_html,
                'pem_folder': 'outbox',
                'pem_account_id': p_mail.pem_account_id.id,
                'mail_type': 'multipart/alternative',
                'date_mail': datetime.now().strftime('%Y-%m-%d'),
                'pem_message_id': p_mail.pem_message_id,
                'conversation_id': case.conversation_id.id,
            }
            mailbox_obj.create(cursor, uid, vals_forward, context=context)
        if email_bcc:
            p_mail = self.browse(cursor, uid, pmail_id, context=context)
            for bcc in email_bcc:
                vals_forward = {
                    'pem_to': bcc,
                    'pem_from': email_from,
                    'pem_subject': p_mail.pem_subject,
                    'pem_body_text': p_mail.pem_body_text,
                    'pem_body_html': p_mail.pem_body_html,
                    'pem_folder': 'outbox',
                    'pem_account_id': p_mail.pem_account_id.id,
                    'mail_type': 'multipart/alternative',
                    'date_mail': datetime.now().strftime('%Y-%m-%d'),
                    'pem_message_id': p_mail.pem_message_id,
                    'conversation_id': case.conversation_id.id,
                }
                mailbox_obj.create(cursor, uid, vals_forward, context=context)

    def create(self, cursor, uid, vals, context=None):
        """If some crm section reply_to has this pem_account create a CRM Case.
        """
        if context is None:
            context = {}
        res_id = super(PoweremailMailboxCRM, self).create(cursor, uid, vals,
                                                          context)
        p_mail = self.browse(cursor, uid, res_id, context=context)
        # This does not work due to smtp's fetch returning always //SEEN
        # if p_mail.state == 'read':
        #     # If downloaded a readed e-mail: do nothing
        #     return res_id
        # If original format mail, use it
        if p_mail.pem_mail_orig:
            mail = qreu.Email.parse(p_mail.pem_mail_orig)
            if mail.is_auto_generated:
                return res_id
            reply_to = [x.lower() for x in mail.recipients.addresses]
        else:
            # If no mail source found, the mail is being sent
            return res_id
        case_obj = self.pool.get('crm.case')
        section_obj = self.pool.get('crm.case.section')
        search_params = [('reply_to', 'in', reply_to)]
        section_id = section_obj.search(cursor, uid, search_params)
        if section_id:
            section_id = section_id[0]
            section = section_obj.browse(cursor, uid, section_id)
            if mail.from_.address == section.reply_to:
                # Ignore mails sent FROM this section
                return res_id

            if self._markdown_inline_images_enabled(cursor, uid):
                body_markdown = self._email_body_as_markdown(p_mail, mail)
                if body_markdown and body_markdown != p_mail.pem_body_text:
                    self.write(cursor, uid, [res_id], {
                        'pem_body_text': body_markdown
                    }, context=context)
                    p_mail = self.browse(cursor, uid, res_id, context=context)

            cases_ids = case_obj.search(cursor, uid, [
                ('conversation_id', '=', p_mail.conversation_id.id)
            ])

            if not cases_ids:
                # Ensure ids from msgid exists in the database
                references_ids = get_cases_ids_from_references(mail.references)
                cases_ids = case_obj.search(cursor, uid, [
                    ('id', 'in', references_ids)
                ])
                cases_no_conversation = case_obj.search(cursor, uid, [
                    ('id', 'in', references_ids),
                    ('conversation_id', '=', False)
                ])
                if cases_no_conversation:
                    # Assign this conversation to all crm that matches
                    case_obj.write(cursor, uid, cases_no_conversation, {
                        'conversation_id': p_mail.conversation_id.id
                    })

            for case_id in cases_ids:
                self.update_case_from_mail(
                    cursor, uid, p_mail.id, case_id, mail, context=context
                )
                # Reread case
                case = case_obj.browse(cursor, uid, case_id, context=context)
                self.forward_case_response(
                    cursor, uid, p_mail.id, case, mail, context=context
                )
                return res_id

            else:
                # If not found a conversation, add new case with email values
                # body_text = quotations.extract_from_plain(
                #     p_mail.pem_body_text)
                self.create_crm_case(
                    cursor, uid, p_mail.id, section_id,
                    body_text=p_mail.pem_body_text
                )

        return res_id


PoweremailMailboxCRM()