Files
ACORE/apps/mail_connector/imap_client.py
T
lokses c6684721e7 mail: batched preview FETCH + одна IMAP-сессия на чтение письма
Убраны ~40 round-trip на список; unread через STATUS; открытие письма
без второго LOGIN.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-22 15:12:48 +03:00

696 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""IMAP-клиент для списка/чтения писем (stdlib imaplib + email)."""
from __future__ import annotations
import email
import imaplib
import logging
import quopri
import re
import ssl
from dataclasses import dataclass
from datetime import datetime
from email.header import decode_header
from email.utils import parseaddr, parsedate_to_datetime
from html import unescape
from apps.mail_connector.mail_settings import MailSettings
logger = logging.getLogger(__name__)
PORTAL_FOLDERS = {
'inbox': 'INBOX',
'sent': 'Sent',
'drafts': 'Drafts',
'trash': 'Trash',
}
class MailConnectionError(Exception):
pass
def _imap_quote(value: str) -> str:
"""Безопасная строка для IMAP SEARCH (без кавычек/управляющих)."""
return value.replace('\\', ' ').replace('"', ' ').replace('\r', ' ').replace('\n', ' ').strip()
def _looks_like_domain(q: str) -> str | None:
"""Вернуть domain.com если запрос похож на домен / @домен."""
domain = (q or '').strip().lstrip('@').lower()
if (
domain
and '.' in domain
and ' ' not in domain
and '@' not in domain
and re.fullmatch(r'[a-z0-9][a-z0-9.-]*\.[a-z]{2,}', domain)
):
return domain
return None
def build_search_strategies(search_text: str = '', *, starred_only: bool = False) -> list[str]:
"""
Несколько вариантов IMAP SEARCH — от простого к узкому.
Глубокий вложенный OR + BODY на Carbonio/Zimbra часто даёт NO/пусто.
Плоский OR (как Roundcube) и TEXT работают стабильнее.
"""
prefix = 'FLAGGED ' if starred_only else ''
q = _imap_quote(search_text or '')
if not q:
return [f'{prefix}ALL'.strip()]
quoted = f'"{q}"'
strategies = [
# 1) TEXT — заголовки + тело (лучше для домена в адресе)
f'{prefix}TEXT {quoted}'.strip(),
# 2) плоский OR по ключевым полям (без BODY)
f'{prefix}OR OR OR SUBJECT {quoted} FROM {quoted} TO {quoted} CC {quoted}'.strip(),
# 3) только адреса
f'{prefix}OR FROM {quoted} TO {quoted}'.strip(),
# 4) тема
f'{prefix}SUBJECT {quoted}'.strip(),
]
domain = _looks_like_domain(q)
if domain:
at_q = f'"@{domain}"'
dom_q = f'"{domain}"'
strategies[0:0] = [
f'{prefix}OR OR FROM {at_q} TO {at_q} CC {at_q}'.strip(),
f'{prefix}OR OR HEADER From {dom_q} HEADER To {dom_q} HEADER Cc {dom_q}'.strip(),
f'{prefix}TEXT {at_q}'.strip(),
]
return strategies
def build_search_criteria(search_text: str = '', *, starred_only: bool = False) -> str:
"""Основной критерий (первый из стратегий) — для тестов/совместимости."""
return build_search_strategies(search_text, starred_only=starred_only)[0]
def _header_matches_query(subject: str, sender: str, query: str, from_addr: str = '') -> bool:
"""Локальная проверка: тема / from / email / домен."""
q = (query or '').strip().lower()
if not q:
return True
hay = f'{subject or ""} {sender or ""} {from_addr or ""}'.lower()
if q in hay:
return True
domain = _looks_like_domain(q)
if domain and domain in hay:
return True
if q.startswith('@') and q[1:] in hay:
return True
if '@' in q and ' ' not in q:
if q in hay:
return True
_, _, dom = q.partition('@')
if dom and dom in hay:
return True
return False
@dataclass
class MailSummary:
uid: str
subject: str
sender: str
date: datetime | None
preview: str
is_unread: bool
from_addr: str = '' # raw email for search/domain match
def _decode_mime(value: str | None) -> str:
if not value:
return ''
parts = []
for chunk, charset in decode_header(value):
if isinstance(chunk, bytes):
parts.append(chunk.decode(charset or 'utf-8', errors='replace'))
else:
parts.append(str(chunk))
return ''.join(parts).strip()
def _decode_bytes(payload: bytes, charset: str | None) -> str:
for enc in (charset, 'utf-8', 'cp1251', 'latin-1'):
if not enc:
continue
try:
return payload.decode(enc)
except (LookupError, UnicodeDecodeError):
continue
return payload.decode('utf-8', errors='replace')
def _part_payload_text(part: email.message.Message) -> str:
"""Декодировать тело MIME-части (base64 / quoted-printable → unicode)."""
payload = part.get_payload(decode=True)
if payload is None:
raw = part.get_payload()
if isinstance(raw, str):
return raw
return ''
if not isinstance(payload, (bytes, bytearray)):
return str(payload)
return _decode_bytes(bytes(payload), part.get_content_charset())
def _html_to_visible_text(html: str) -> str:
"""HTML → читаемый текст (без CSS/скриптов)."""
text = html or ''
text = re.sub(r'(?is)<(script|style|head|title|noscript)[^>]*>.*?</\1>', ' ', text)
text = re.sub(r'(?is)<!--.*?-->', ' ', text)
text = re.sub(r'(?i)<br\s*/?>', '\n', text)
text = re.sub(r'(?i)</p\s*>', '\n\n', text)
text = re.sub(r'(?i)</div\s*>', '\n', text)
text = re.sub(r'(?i)</tr\s*>', '\n', text)
text = re.sub(r'(?i)</li\s*>', '\n', text)
text = re.sub(r'(?s)<[^>]+>', ' ', text)
text = unescape(text)
text = text.replace('\xa0', ' ')
text = re.sub(r'[ \t]+\n', '\n', text)
text = re.sub(r'\n{3,}', '\n\n', text)
text = re.sub(r'[ \t]{2,}', ' ', text)
return text.strip()
def _sanitize_email_html(html: str) -> str:
"""Минимальная санация для iframe sandbox (без script/handlers)."""
text = html or ''
text = re.sub(r'(?is)<script[^>]*>.*?</script>', '', text)
text = re.sub(r'(?is)<iframe[^>]*>.*?</iframe>', '', text)
text = re.sub(r'(?is)<object[^>]*>.*?</object>', '', text)
text = re.sub(r'(?is)<embed[^>]*>', '', text)
text = re.sub(r'(?is)\s+on[a-z]+\s*=\s*("[^"]*"|\'[^\']*\'|[^\s>]+)', '', text)
text = re.sub(r'(?i)javascript:', '', text)
text = re.sub(r'(?i)vbscript:', '', text)
# относительные cid: картинки не загрузятся — ок
return text
def _maybe_decode_quoted_printable(raw: str) -> str:
if not raw:
return ''
# типичные маркеры QP в превью BODY.PEEK[TEXT]
if '=3D' in raw or '=\r\n' in raw or '=\n' in raw or re.search(r'=[0-9A-F]{2}', raw):
try:
return quopri.decodestring(raw.encode('latin-1', errors='replace')).decode('utf-8', errors='replace')
except Exception:
return raw
return raw
def _clean_preview_snippet(raw: str, limit: int = 140) -> str:
text = _maybe_decode_quoted_printable(raw or '')
if 'Content-Type:' in text[:500] or text.lstrip().startswith('--') or 'multipart message' in text[:120].lower():
parts = re.split(r'\r?\n\r?\n', text, maxsplit=3)
# взять последний «похожий на тело» кусок
for part in reversed(parts):
cand = part.strip()
if not cand or cand.lower().startswith('content-'):
continue
if cand.startswith('--'):
continue
text = cand
break
text = re.sub(r'(?im)^(Content-|MIME-|--+|_NextPart).*$', ' ', text)
low = text.lstrip()[:300].lower()
if (
low.startswith('<!doctype')
or low.startswith('<html')
or '<body' in low
or 'text/css' in low
or '@media' in low
or '@font-face' in low
):
text = _html_to_visible_text(text)
elif '<' in text[:100] and '>' in text[:300]:
text = _html_to_visible_text(text)
text = ' '.join(text.split())
if len(text) > limit:
return text[: limit - 1].rstrip() + '…'
return text
def _preview_from_peek_bytes(chunk: bytes, limit: int = 140) -> str:
"""Превью из куска RFC822 / TEXT: сначала полный MIME-парс, иначе эвристика."""
if not chunk:
return ''
# 1) Попробовать как начало полного письма (BODY.PEEK[]<N>)
try:
msg = email.message_from_bytes(chunk)
# если есть Content-Type или multipart — парсер понял структуру
if msg.get_content_maintype() != 'text' or msg.get_content_type() == 'text/html' or msg.is_multipart():
preview = _preview_from_message(msg)
if preview:
return preview[:limit] + ('…' if len(preview) > limit else '')
# text/plain без multipart
if msg.get_content_type() == 'text/plain':
text = _part_payload_text(msg)
if text.strip():
return ' '.join(text.split())[:limit]
except Exception:
pass
# 2) Обернуть TEXT-peek (без заголовков письма) в минимальный MIME
try:
raw = chunk.decode('utf-8', errors='replace')
except Exception:
raw = chunk.decode('latin-1', errors='replace')
if raw.lstrip().startswith('--') or 'Content-Type:' in raw[:800]:
wrapped = (
'MIME-Version: 1.0\r\n'
'Content-Type: multipart/mixed; boundary="ACORE-PEEK"\r\n\r\n'
'--ACORE-PEEK\r\n'
f'{raw}\r\n'
'--ACORE-PEEK--\r\n'
).encode('utf-8', errors='replace')
try:
msg = email.message_from_bytes(wrapped)
preview = _preview_from_message(msg)
if preview:
return preview[:limit] + ('…' if len(preview) > limit else '')
except Exception:
pass
return _clean_preview_snippet(raw, limit=limit)
def _extract_bodies(msg: email.message.Message) -> tuple[str | None, str | None]:
"""Вернуть (text/plain, text/html) с декодированием."""
text_plain = None
text_html = None
if msg.is_multipart():
for part in msg.walk():
if part.is_multipart():
continue
disp = (part.get_content_disposition() or '').lower()
if disp == 'attachment':
continue
ctype = (part.get_content_type() or '').lower()
if ctype == 'text/plain' and text_plain is None:
text_plain = _part_payload_text(part)
elif ctype == 'text/html' and text_html is None:
text_html = _part_payload_text(part)
else:
ctype = (msg.get_content_type() or '').lower()
raw = _part_payload_text(msg)
if ctype == 'text/html':
text_html = raw
else:
text_plain = raw
return text_plain, text_html
def _preview_from_message(msg: email.message.Message, limit: int = 140) -> str:
text_plain, text_html = _extract_bodies(msg)
if text_plain and text_plain.strip():
return ' '.join(text_plain.split())[:limit]
if text_html:
return _html_to_visible_text(text_html)[:limit]
return ''
def _message_content(msg: email.message.Message) -> tuple[str, str | None, bool]:
"""(body_text, body_html_sanitized|None, is_html)."""
text_plain, text_html = _extract_bodies(msg)
plain = (text_plain or '').strip()
html_raw = (text_html or '').strip()
if html_raw:
sanitized = _sanitize_email_html(html_raw)
# если plain пустой или это «отписка одной строкой» — показываем HTML
if not plain or len(plain) < 40:
return _html_to_visible_text(html_raw), sanitized, True
return plain, sanitized, True
return plain, None, False
def _iter_file_parts(msg: email.message.Message):
"""Вложения: disposition=attachment или файл с именем не text/*."""
index = 0
for part in msg.walk():
if part.is_multipart():
continue
filename = part.get_filename()
disp = (part.get_content_disposition() or '').lower()
ctype = (part.get_content_type() or '').lower()
is_file = disp == 'attachment' or (
bool(filename) and ctype not in ('text/plain', 'text/html')
)
if not is_file:
continue
name = _decode_mime(filename) or f'attachment-{index + 1}'
payload = part.get_payload(decode=True) or b''
yield index, name, payload, ctype or 'application/octet-stream'
index += 1
def extract_reply_address(header: str) -> str:
_name, addr = parseaddr(header or '')
return (addr or '').strip()
def display_address(header: str) -> str:
"""Имя отправителя для списка (без угловых скобок, если есть display-name)."""
name, addr = parseaddr(header or '')
name = _decode_mime(name).strip() if name else ''
if name:
# убрать кавычки вокруг имени
if len(name) >= 2 and name[0] == name[-1] and name[0] in '\"\'':
name = name[1:-1].strip()
return name or addr or '—'
return (addr or (header or '').strip() or '—')
def reply_subject(subject: str) -> str:
text = (subject or '').strip()
if text.lower().startswith('re:'):
return text
return f'Re: {text}' if text else 'Re:'
def _attachment_summaries(msg: email.message.Message) -> list[dict]:
items = []
for index, name, payload, _ctype in _iter_file_parts(msg):
items.append({'index': index, 'name': name, 'size': len(payload)})
return items
class ImapMailClient:
def __init__(self, settings: MailSettings, username: str, password: str):
self.settings = settings
self.username = username
self.password = password
self._conn: imaplib.IMAP4 | imaplib.IMAP4_SSL | None = None
self._selected_folder: str | None = None
def __enter__(self):
self.connect()
return self
def __exit__(self, *args):
self.close()
def connect(self) -> None:
s = self.settings
try:
if s.use_ssl:
ctx = ssl.create_default_context()
if not s.verify_ssl:
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
self._conn = imaplib.IMAP4_SSL(s.imap_host, s.imap_port, ssl_context=ctx)
else:
self._conn = imaplib.IMAP4(s.imap_host, s.imap_port)
# imaplib по умолчанию ascii — пароли/логины с Unicode дают UnicodeEncodeError.
self._conn._encoding = 'utf-8'
typ, data = self._conn.login(self.username, self.password)
if typ != 'OK':
detail = data[0].decode() if data and isinstance(data[0], (bytes, bytearray)) else (data[0] if data else 'IMAP login failed')
raise MailConnectionError(str(detail))
except MailConnectionError:
self.close()
raise
except imaplib.IMAP4.error as e:
self.close()
raise MailConnectionError(str(e)) from e
except (OSError, UnicodeEncodeError, ssl.SSLError, TimeoutError) as e:
self.close()
raise MailConnectionError(str(e)) from e
except Exception as e:
self.close()
raise MailConnectionError(str(e)) from e
def close(self) -> None:
if self._conn:
try:
self._conn.logout()
except Exception:
pass
self._conn = None
self._selected_folder = None
def _select_folder(self, folder: str) -> None:
assert self._conn
if self._selected_folder == folder:
return
typ, _ = self._conn.select(f'"{folder}"', readonly=False)
if typ != 'OK':
raise MailConnectionError(f'Папка недоступна: {folder}')
self._selected_folder = folder
def _uid_search(self, criteria: str) -> list[bytes] | None:
"""Выполнить UID SEARCH. None = ошибка сервера; [] = пусто."""
assert self._conn
try:
# UTF-8 для кириллицы в теме/адресах
if any(ord(c) > 127 for c in criteria):
typ, data = self._conn.uid('SEARCH', 'UTF-8', criteria)
else:
typ, data = self._conn.uid('SEARCH', None, criteria)
except (imaplib.IMAP4.error, OSError) as e:
logger.info('IMAP SEARCH failed (%s): %s', criteria[:80], e)
return None
if typ != 'OK':
logger.info('IMAP SEARCH not OK (%s): %s %s', criteria[:80], typ, data)
return None
if not data or not data[0]:
return []
return data[0].split()
def _search_uids(self, search_text: str, *, starred_only: bool) -> list[bytes]:
"""Пробуем стратегии SEARCH; при пустоте/ошибке — ALL (локальный фильтр дальше)."""
strategies = build_search_strategies(search_text, starred_only=starred_only)
for crit in strategies:
uids = self._uid_search(crit)
if uids is None:
continue
if uids:
return list(reversed(uids))
# ничего не нашли сервером — отдаём все UID, отфильтруем по заголовкам
if search_text.strip():
uids = self._uid_search('FLAGGED' if starred_only else 'ALL')
if uids:
return list(reversed(uids))
return []
def list_messages(
self,
folder_key: str = 'inbox',
*,
limit: int = 40,
offset: int = 0,
search_text: str = '',
starred_only: bool = False,
) -> tuple[list[MailSummary], bool]:
"""Список писем (новые сверху). Возвращает (messages, has_more)."""
assert self._conn
offset = max(0, int(offset or 0))
limit = max(1, min(int(limit or 40), 100))
query = (search_text or '').strip()
if starred_only:
imap_folder = PORTAL_FOLDERS['inbox']
self._select_folder(imap_folder)
else:
imap_folder = PORTAL_FOLDERS.get(folder_key, 'INBOX')
self._select_folder(imap_folder)
all_uids = self._search_uids(query, starred_only=starred_only)
if not all_uids:
return [], False
# Локальный фильтр по заголовкам (тема/from/домен), если есть запрос:
# серверный SEARCH часто игнорит домен или отдаёт пусто на Carbonio.
if query:
# сканируем пачками до разумного лимита, чтобы набрать страницу
scan_cap = min(len(all_uids), max(500, offset + limit * 10))
matched: list[bytes] = []
chunk_size = 100
for i in range(0, scan_cap, chunk_size):
chunk = all_uids[i:i + chunk_size]
headers = self._fetch_headers(chunk)
for uid in chunk:
key = uid.decode() if isinstance(uid, bytes) else str(uid)
summary = headers.get(key)
if summary and _header_matches_query(
summary.subject, summary.sender, query, summary.from_addr,
):
matched.append(uid)
if len(matched) >= offset + limit + 20:
break
page_source = matched
else:
page_source = all_uids
has_more = (offset + limit) < len(page_source)
page_uids = page_source[offset: offset + limit]
if not page_uids:
return [], False
by_uid = self._fetch_headers(page_uids)
summaries = [
by_uid[u.decode() if isinstance(u, bytes) else str(u)]
for u in page_uids
if (u.decode() if isinstance(u, bytes) else str(u)) in by_uid
]
previews = self._fetch_previews(page_uids)
for summary in summaries:
summary.preview = previews.get(summary.uid, '')
return summaries, has_more
def _fetch_previews(self, page_uids: list[bytes]) -> dict[str, str]:
"""Один UID FETCH на всю страницу вместо N отдельных BODY.PEEK."""
assert self._conn
if not page_uids:
return {}
uid_set = b','.join(page_uids).decode()
try:
# 4 KB хватает для сниппета; батч = 1 round-trip вместо ~40
typ, fetched = self._conn.uid('fetch', uid_set, '(BODY.PEEK[]<0.4096>)')
except Exception:
logger.debug('batched preview fetch failed', exc_info=True)
return {}
if typ != 'OK' or not fetched:
return {}
previews: dict[str, str] = {}
for item in fetched:
if not isinstance(item, tuple) or len(item) < 2:
continue
meta = item[0].decode() if isinstance(item[0], bytes) else str(item[0])
m = re.search(r'UID (\d+)', meta)
if not m:
continue
chunk = item[1] if isinstance(item[1], (bytes, bytearray)) else b''
if not chunk:
continue
try:
previews[m.group(1)] = _preview_from_peek_bytes(bytes(chunk))
except Exception:
logger.debug('preview parse failed uid=%s', m.group(1), exc_info=True)
return previews
def _fetch_headers(self, page_uids: list[bytes]) -> dict[str, MailSummary]:
assert self._conn
if not page_uids:
return {}
uid_set = b','.join(page_uids).decode()
typ, fetched = self._conn.uid(
'fetch', uid_set, '(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)] FLAGS)',
)
if typ != 'OK' or not fetched:
return {}
by_uid: dict[str, MailSummary] = {}
for item in fetched:
if not isinstance(item, tuple) or len(item) < 2:
continue
meta = item[0].decode() if isinstance(item[0], bytes) else str(item[0])
m = re.search(r'UID (\d+)', meta)
if not m:
continue
uid = m.group(1)
flags = meta
header_bytes = item[1] if isinstance(item[1], bytes) else b''
msg = email.message_from_bytes(header_bytes)
date_raw = msg.get('Date')
try:
dt = parsedate_to_datetime(date_raw) if date_raw else None
except (TypeError, ValueError):
dt = None
from_raw = _decode_mime(msg.get('From')) or ''
_name, addr = parseaddr(from_raw)
by_uid[uid] = MailSummary(
uid=uid,
subject=_decode_mime(msg.get('Subject')) or '(без теми)',
sender=display_address(from_raw),
date=dt,
preview='',
is_unread='\\Seen' not in flags,
from_addr=(addr or '').strip().lower(),
)
return by_uid
def fetch_message(self, uid: str, folder_key: str = 'inbox') -> dict:
assert self._conn
uid = (uid or '').strip()
if not uid.isdigit():
raise MailConnectionError('Лист не знайдено')
imap_folder = PORTAL_FOLDERS.get(folder_key, 'INBOX')
self._select_folder(imap_folder)
typ, data = self._conn.uid('fetch', uid, '(RFC822 FLAGS)')
if typ != 'OK' or not data or not data[0]:
raise MailConnectionError('Лист не знайдено')
raw = data[0][1]
msg = email.message_from_bytes(raw)
flags = data[0][0].decode() if isinstance(data[0][0], bytes) else ''
if '\\Seen' not in flags:
self._conn.uid('store', uid, '+FLAGS', '(\\Seen)')
body_text, body_html, is_html = _message_content(msg)
return {
'uid': uid,
'subject': _decode_mime(msg.get('Subject')) or '(без теми)',
'sender': _decode_mime(msg.get('From')) or '—',
'to': _decode_mime(msg.get('To')) or '—',
'date': _decode_mime(msg.get('Date')) or '',
'body': body_text,
'body_html': body_html,
'is_html': is_html,
'attachments': _attachment_summaries(msg),
'reply_to': extract_reply_address(
_decode_mime(msg.get('Reply-To')) or _decode_mime(msg.get('From'))
),
'reply_subject': reply_subject(_decode_mime(msg.get('Subject')) or ''),
}
def fetch_attachment(self, uid: str, folder_key: str, index: int) -> tuple[str, bytes, str]:
assert self._conn
uid = (uid or '').strip()
if not uid.isdigit():
raise MailConnectionError('Лист не знайдено')
imap_folder = PORTAL_FOLDERS.get(folder_key, 'INBOX')
self._select_folder(imap_folder)
typ, data = self._conn.uid('fetch', uid, '(RFC822)')
if typ != 'OK' or not data or not data[0]:
raise MailConnectionError('Лист не знайдено')
raw = data[0][1]
msg = email.message_from_bytes(raw)
parts = list(_iter_file_parts(msg))
if index < 0 or index >= len(parts):
raise MailConnectionError('Файл не знайдено')
_i, name, payload, ctype = parts[index]
return name, payload, ctype
def unread_count(self, folder_key: str = 'inbox') -> int:
assert self._conn
imap_folder = PORTAL_FOLDERS.get(folder_key, 'INBOX')
# STATUS не требует SELECT и не сбрасывает текущую папку.
try:
typ, data = self._conn.status(f'"{imap_folder}"', '(UNSEEN)')
if typ == 'OK' and data and data[0]:
raw = data[0] if isinstance(data[0], (bytes, bytearray)) else str(data[0]).encode()
m = re.search(rb'UNSEEN\s+(\d+)', raw)
if m:
return int(m.group(1))
except Exception:
logger.debug('STATUS UNSEEN failed, fallback SEARCH', exc_info=True)
self._select_folder(imap_folder)
typ, data = self._conn.uid('search', None, 'UNSEEN')
if typ != 'OK' or not data or not data[0]:
return 0
return len(data[0].split())