Source code for parsedmarc

# -*- coding: utf-8 -*-

"""A Python package for parsing DMARC reports"""

from __future__ import annotations

import binascii
import email
import email.errors
import email.message
import email.utils
import functools
import hashlib
import ipaddress
import json
import logging
import mailbox
import os
import quopri
import re
import shutil
import tempfile
import time
import traceback
import xml.parsers.expat as expat
import zipfile
import zlib
from base64 import b64decode
from collections import deque
from collections.abc import Callable, Sequence
from csv import DictWriter
from dataclasses import dataclass, field
from datetime import date, datetime, timedelta, timezone, tzinfo
from io import BytesIO, StringIO
from typing import (
    Any,
    BinaryIO,
    cast,
)

import lxml.etree as etree
import mailparser
import xmltodict
from expiringdict import ExpiringDict
from mailsuite.smtp import send_email
from mailsuite.utils import MessageLimitExceeded, check_message_limits
from tqdm import tqdm

from parsedmarc.attempts import MAX_PARSE_ATTEMPTS, AttemptStore, attempt_key
from parsedmarc.config import (
    IP_ADDRESS_CACHE,
    REVERSE_DNS_MAP,
    SEEN_AGGREGATE_REPORT_IDS,
    ParserConfig,
)
from parsedmarc.constants import (
    DEFAULT_DNS_MAX_RETRIES,
    DEFAULT_DNS_TIMEOUT,
    MAX_FAILED_REVERSE_DNS_SECONDS_PER_REPORT,
    MAX_REVERSE_DNS_SECONDS_PER_REPORT,
    SLOW_REVERSE_DNS_SECONDS,
    MAX_MESSAGE_COUNT,
    MAX_NORMALIZED_RECORDS,
    MAX_REPORT_FUTURE_DAYS,
    MAX_REPORT_SPAN_DAYS,
    MAX_XML_DEPTH,
    MAX_XML_NODES,
    MAX_XML_TAG_LENGTH,
    __version__,
)
from parsedmarc.log import logger
from parsedmarc.mail import (
    GmailConnection,
    IMAPConnection,
    MailboxConnection,
    MaildirConnection,
    MSGraphConnection,
)
from parsedmarc.types import (
    AggregateReport,
    FailureReport,
    ForensicReport as ForensicReport,
    ParsedReport,
    ParsingResults,
    ReportType,
    SMTPTLSReport,
)
from parsedmarc.utils import (
    IPAddressInfo,
    convert_outlook_msg,
    get_base_domain,
    get_filename_safe_string,
    get_ip_address_info,
    human_timestamp_to_datetime,
    human_timestamp_to_unix_timestamp,
    is_outlook_msg,
    parse_email,
    sanitize_for_log,
    timestamp_to_human,
)

logger.debug(f"parsedmarc v{__version__}")

# A message/feedback-report part is a MIME body part, so its lines end with
# CRLF per RFC 5322 §2.1. In re.MULTILINE, ``$`` matches before the LF but not
# before the CR, so the value must exclude the line ending explicitly rather
# than relying on ``.`` — otherwise every field value keeps a trailing CR.
feedback_report_regex = re.compile(r"^([\w\-]+): ([^\r\n]+)\r?$", re.MULTILINE)
xml_header_regex = re.compile(r"^<\?xml .*?>", re.MULTILINE)
# Every pattern here runs on attacker-controlled input; the next two were
# rewritten to match in linear time (GHSA-4h9r-c435-j834, GHSA-7grx-mj6g-qxmc):
# a repeated character class excludes the delimiter it stops on, adjacent
# quantifiers never overlap, and text_report_regex is anchored to a line start,
# so each candidate match is scanned once instead of being rescanned from every
# offset on every backtrack. See "Regular expressions on untrusted input" in
# AGENTS.md.
xml_schema_regex = re.compile(r"</?xs:schema[^<>]*>")
text_report_regex = re.compile(
    r"^[ \t]*([A-Za-z]+(?: [A-Za-z]+)*):[ \t]+([^\r\n]+)", re.MULTILINE
)
# Captures the value of any xmlns (default or prefixed) declaration so the
# RFC 9990 namespace can be detected before xmltodict drops it.
xml_namespace_regex = re.compile(
    r"""xmlns(?::[a-zA-Z_][\w.-]*)?\s*=\s*["']([^"']+)["']"""
)

# The XML namespace assigned to DMARC aggregate reports by RFC 9990.
RFC_9990_NAMESPACE = "urn:ietf:params:xml:ns:dmarc-2.0"

# PolicyOverrideType enumeration from RFC 9990. Compared to RFC 7489,
# `policy_test_mode` was added (emitted when t=y suppresses enforcement)
# and `forwarded` / `sampled_out` were removed.
RFC_9990_POLICY_OVERRIDE_TYPES = frozenset(
    {
        "local_policy",
        "mailing_list",
        "other",
        "policy_test_mode",
        "trusted_forwarder",
    }
)
RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES = frozenset({"forwarded", "sampled_out"})

MAGIC_ZIP = b"\x50\x4b\x03\x04"
MAGIC_GZIP = b"\x1f\x8b"
MAGIC_XML = b"\x3c\x3f\x78\x6d\x6c\x20"
MAGIC_XML_TAG = b"\x3c"  # '<' - XML starting with an element tag (no declaration)
# 0x7B, "{" -- a JSON text that is an object begins with it (RFC 8259).
# Previously written as b"\7b", which Python reads as the octal escape
# \7 (BEL) followed by a literal "b", so the branch never matched real
# JSON; every in-tree caller happened to pre-guard with its own zip/gzip
# or "{" check, which masked it.
MAGIC_JSON = b"\x7b"

# Maximum size, in bytes, of the data extracted from one compressed report
# attachment, and of an uncompressed report or report file measured as UTF-8
# (_exceeds_size_cap()), so a non-ASCII text cannot be four times the cap in
# bytes. Real reports compress at roughly 10:1, but the attachment's
# content is chosen by whoever sent it, and deflate (used by both gzip and
# zip) reaches about 1000:1 on degenerate input, so a 32 KB attachment from
# any sender a monitored mailbox accepts could inflate to 32 MB -- and the
# decoded str coexists with the decompressed bytes, so the peak is roughly
# twice that. Deliberately not configurable: an aggregate report is a day of
# one reporter's traffic, the largest sample is 909 KB, and the XML node
# limit (MAX_XML_NODES, 1,000,000 nodes at the samples' 35 nodes per KB)
# already rejects a report of more than about 28 MB (GHSA-q7gm-8cj9-94hr).
MAX_DECOMPRESSED_REPORT_SIZE = 32 * 1024 * 1024

# Per-message count of consecutive failed saves, keyed on
# ``(reports_folder, str(message_uid))``. Populated only when
# ``get_dmarc_reports_from_mailbox()`` is given a ``save_callback`` that
# reports a batch as unsaved; a message whose count exceeds
# ``max_unsaved_retries`` is moved to ``{archive_folder}/Unsaved`` instead
# of being retried forever, and its entry is dropped. A successful save
# clears the entries for that batch's messages.
#
# Process-local and deliberately not persisted: a restart re-attempts every
# message still in the reports folder, which is the safe direction (retry
# rather than shelve). The key carries no connection identity, so a library
# caller processing two connections that share a folder name (two IMAP
# servers, both "INBOX") in one process could collide counters if UIDs
# happen to match; the CLI uses one connection per process. It is not a
# ``ParserConfig`` field for the same reason ``batch_size`` and the
# ``delete`` flags are not -- it governs mailbox orchestration, not
# parsing.
_FAILED_SAVE_ATTEMPTS: dict[tuple[str, str], int] = {}

EMAIL_SAMPLE_CONTENT_TYPES = (
    "text/rfc822",
    "text-rfc-822",
    "text/rfc822-headers",
    "text/rfc-822-headers",
    "message/rfc822",
    "message/rfc-822",
    "message/rfc822-headers",
    "message/rfc-822-headers",
)


[docs] class ParserError(RuntimeError): """Raised whenever the parser fails for some reason"""
[docs] class InvalidDMARCReport(ParserError): """Raised when an invalid DMARC report is encountered"""
[docs] class InvalidSMTPTLSReport(ParserError): """Raised when an invalid SMTP TLS report is encountered"""
[docs] class ParseTimeoutError(InvalidDMARCReport): """Returned in place of a report whose parsing exceeded ``parse_timeout`` An ``InvalidDMARCReport``, so a timed-out message is handled like any other message that cannot be parsed: the mbox loop warns and continues, and the mailbox loop moves it to the ``Invalid`` archive folder. It is the value for every timed-out input, including one that would have been an SMTP TLS report or not a report at all, since a parse that was stopped cannot tell what the input held. """
[docs] class InvalidAggregateReport(InvalidDMARCReport): """Raised when an invalid DMARC aggregate report is encountered"""
[docs] class InvalidFailureReport(InvalidDMARCReport): """Raised when an invalid DMARC failure report is encountered"""
# Backward-compatible alias InvalidForensicReport = InvalidFailureReport _DOCTYPE_ERROR = "Aggregate reports must not contain a DOCTYPE declaration" class _XMLDoctypeFound(Exception): """Raised by :class:`_NoDoctypeExpat` parsers as soon as expat reports the start of a ``<!DOCTYPE`` declaration.""" def _raise_doctype_found(*args: Any) -> None: raise _XMLDoctypeFound(_DOCTYPE_ERROR) class _NoDoctypeExpat: """A minimal stand-in for the ``xml.parsers.expat`` module, passed to ``xmltodict.parse(expat=...)``, whose parsers abort on any DOCTYPE. An internal DTD subset can declare default attributes (``<!ATTLIST z a0 CDATA "v" ... aK ...>``) that expat then reports on every matching element, so K declared defaults on N empty elements make xmltodict build K x N dict entries from a few kilobytes of input (GHSA-m47j-wp78-fqjg). Entity expansion is already off (xmltodict's ``disable_entities=True``) but does not cover this. DMARC aggregate reports never carry a DOCTYPE, so the declaration is rejected outright, before expat reads any of the subset. ``xmltodict.parse()`` only ever calls ``ParserCreate()`` on the module it is given. """ @staticmethod def ParserCreate(*args: Any, **kwargs: Any) -> Any: parser = expat.ParserCreate(*args, **kwargs) parser.StartDoctypeDeclHandler = _raise_doctype_found return _LimitedExpatParser(parser) class _XMLLimitExceeded(Exception): """Raised by :class:`_LimitedExpatParser` as soon as a document goes over ``MAX_XML_DEPTH`` or ``MAX_XML_NODES``.""" class _LimitedExpatParser: """Wraps an expat parser so that the element handlers xmltodict installs count nesting depth and nodes (elements plus attributes), and raise :class:`_XMLLimitExceeded` over ``MAX_XML_DEPTH`` or ``MAX_XML_NODES``. xmltodict keeps one stack entry per open element, so a document of millions of unclosed ``<a>`` tags, or of millions of siblings, could cost gigabytes before expat reported the error at the end of the input (GHSA-q7gm-8cj9-94hr). The check runs before xmltodict's handler, so the tree never grows past the limits. pyexpat parser objects cannot be subclassed, and xmltodict assigns its handlers after ``ParserCreate()`` returns, so this proxy wraps the handlers in ``__setattr__`` and forwards every other attribute. The wrappers keep their counts in a :class:`_XMLCounts`, not on the proxy: a wrapper referring to the proxy would close a reference cycle through the parser and keep xmltodict's whole tree alive until the cyclic garbage collector ran, doubling peak memory across the two parses. """ def __init__(self, parser: Any) -> None: object.__setattr__(self, "_parser", parser) object.__setattr__(self, "_counts", _XMLCounts()) def __getattr__(self, name: str) -> Any: return getattr(self._parser, name) def __setattr__(self, name: str, value: Any) -> None: if name == "StartElementHandler": value = _limit_start(value, self._counts) elif name == "EndElementHandler": value = _limit_end(value, self._counts) setattr(self._parser, name, value) class _XMLCounts: """The open-element depth and node count of one expat parse.""" def __init__(self) -> None: self.depth = 0 self.nodes = 0 def _limit_start(handler: Callable[..., Any], counts: _XMLCounts) -> Callable[..., Any]: """Wrap an expat start-element handler so that it counts depth and nodes in ``counts`` and raises :class:`_XMLLimitExceeded` over the limits before ``handler`` runs.""" def start(name: str, attributes: list[str]) -> Any: depth = counts.depth + 1 # xmltodict sets ordered_attributes, so expat passes attributes as # a flat [name, value, name, value, ...] list. nodes = counts.nodes + 1 + len(attributes) // 2 if depth > MAX_XML_DEPTH: raise _XMLLimitExceeded( f"elements nested more than {MAX_XML_DEPTH} levels deep" ) if nodes > MAX_XML_NODES: raise _XMLLimitExceeded( f"more than {MAX_XML_NODES:,} elements and attributes" ) counts.depth = depth counts.nodes = nodes return handler(name, attributes) return start def _limit_end(handler: Callable[..., Any], counts: _XMLCounts) -> Callable[..., Any]: """Wrap an expat end-element handler so that it decrements the depth in ``counts``.""" def end(name: str) -> Any: counts.depth -= 1 return handler(name) return end class _LxmlNodeCounter: """An lxml parser target that counts elements, attributes and namespace declarations without building a tree, and raises :class:`_XMLLimitExceeded` past ``MAX_XML_NODES``. It runs before the lxml recovery parse, whose tree would otherwise cost gigabytes for a wide document behind a malformed XML declaration (GHSA-q7gm-8cj9-94hr). Depth is not counted here: libxml2 stops at 256 levels without ``huge_tree``, and the second ``xmltodict.parse()`` enforces ``MAX_XML_DEPTH`` on the recovered document. """ def __init__(self) -> None: self.nodes = 0 def _add(self, count: int) -> None: self.nodes += count if self.nodes > MAX_XML_NODES: raise _XMLLimitExceeded( f"more than {MAX_XML_NODES:,} elements and attributes" ) def start(self, tag: str, attrib: dict[str, str]) -> None: self._add(1 + len(attrib)) def start_ns(self, prefix: str | None, uri: str) -> None: # xmltodict, which does not process namespaces, sees each xmlns # declaration as an attribute. self._add(1) def end(self, tag: str) -> None: pass def close(self) -> int: return self.nodes def _parse_xml_with_recovery(xml: bytes) -> Any: """Parse ``xml`` with lxml in recovery mode and return the tree, after a counting pass that raises :class:`_XMLLimitExceeded` over ``MAX_XML_NODES``, so the tree is only built for a document within the limit.""" etree.parse( BytesIO(xml), etree.XMLParser( recover=True, resolve_entities=False, target=_LxmlNodeCounter() ), ) return etree.parse( BytesIO(xml), etree.XMLParser(recover=True, resolve_entities=False) ) def _resolve_config( config: ParserConfig | None, *, offline: bool = False, ip_db_path: str | None = None, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, nameservers: list[str] | None = None, dns_timeout: float = DEFAULT_DNS_TIMEOUT, dns_retries: int = DEFAULT_DNS_MAX_RETRIES, strip_attachment_payloads: bool = False, normalize_timespan_threshold_hours: float = 24.0, ) -> ParserConfig: """Resolve the effective :class:`~parsedmarc.config.ParserConfig` for a public parsing call, from either an explicit ``config`` or the caller's individual option keyword arguments. If ``config`` is not ``None``, it is returned unchanged: per the documented ``config=`` contract, the individual option keyword arguments are ignored in favor of the config's own values, so no merging happens here. Otherwise, a new ``ParserConfig`` is built from the given keyword arguments. Its three cache fields are deliberately *not* left to their ``default_factory`` -- doing so would hand back a config with brand new, empty caches on every call, silently defeating cross-call IP-info caching and aggregate-report dedup. Instead, the module-default caches (:data:`IP_ADDRESS_CACHE`, :data:`SEEN_AGGREGATE_REPORT_IDS`, :data:`REVERSE_DNS_MAP`) are injected explicitly, so kwargs-style calls keep observing and mutating the same shared caches they always have. ``dataclasses.replace`` is deliberately not used here: it would need an existing ``ParserConfig`` to start from, and there isn't one on the kwargs path -- this function's job is to build the first one. ``psl_overrides_path`` / ``psl_overrides_url`` have no corresponding keyword arguments on the public functions (see AGENTS.md's guidance on justifying new config options), so they stay ``None`` on this path; they are only ever set via an explicitly constructed ``config=``. """ if config is not None: return config return ParserConfig( offline=offline, ip_db_path=ip_db_path, always_use_local_files=always_use_local_files, reverse_dns_map_path=reverse_dns_map_path, reverse_dns_map_url=reverse_dns_map_url, nameservers=nameservers, dns_timeout=dns_timeout, dns_retries=dns_retries, strip_attachment_payloads=strip_attachment_payloads, normalize_timespan_threshold_hours=float(normalize_timespan_threshold_hours), ip_address_cache=IP_ADDRESS_CACHE, seen_aggregate_report_ids=SEEN_AGGREGATE_REPORT_IDS, reverse_dns_map=REVERSE_DNS_MAP, ) def _exc_origin(error: BaseException) -> str: """Returns a ``" (raised at <file>:<line>)"`` suffix pointing at where an unexpected exception actually originated, but only when the parsedmarc logger is at ``DEBUG`` level. Returns ``""`` otherwise, so normal output is unchanged. The deepest traceback frame is the line that raised, which is the useful breadcrumb when a catch-all ``except Exception`` turns an unforeseen error into a generic ``Invalid*Report`` -- without it, the message says *what* failed but not *where*. """ if not logger.isEnabledFor(logging.DEBUG): return "" frames = traceback.extract_tb(error.__traceback__) if not frames: return "" last = frames[-1] return f" (raised at {last.filename}:{last.lineno})" def _text(value: Any) -> str | None: """Unwrap a value parsed by xmltodict that may be a dict. RFC 9990 changed several aggregate-report elements (extra_contact_info, error, comment, human_result) to type ``langAttrString``, a string with an optional ``lang`` attribute. xmltodict parses the report without attributes (``xml_attribs=False``), so such an element arrives as a string; a dict now arises only from child elements, and gives its ``#text``. Returns ``None`` for unset values and leaves other scalar shapes untouched. """ if value is None: return None if isinstance(value, dict): text = value.get("#text") return None if text is None else str(text) return value def _normalize_result_word(value: Any) -> Any: """Lowercase a reporter-supplied enum word; RFC 7489 Appendix C and RFC 9990 define result/disposition types as lowercase tokens. Its callers pass strings or ``None`` (``_scalar()`` has already run), and a non-string passes through unchanged. """ return value.lower() if isinstance(value, str) else value def _bucket_interval_by_day( begin: datetime, end: datetime, total_count: int, ) -> list[dict[str, Any]]: """ Split the interval [begin, end) into daily buckets and distribute `total_count` proportionally across those buckets. The function: 1. Identifies each calendar day touched by [begin, end) 2. Computes how many seconds of the interval fall into each day 3. Assigns counts in proportion to those overlaps 4. Ensures the final counts sum exactly to total_count Args: begin: timezone-aware datetime, inclusive start of interval end: timezone-aware datetime, exclusive end of interval total_count: number of messages to distribute Returns: A list of dicts like: { "begin": datetime, "end": datetime, "count": int } """ # --- Input validation ---------------------------------------------------- if begin > end: raise ValueError("begin must be earlier than end") if begin.tzinfo is None or end.tzinfo is None: raise ValueError("begin and end must be timezone-aware") if begin.tzinfo is not end.tzinfo: raise ValueError("begin and end must have the same tzinfo") if total_count < 0: raise ValueError("total_count must be non-negative") # --- Short-circuit trivial cases ----------------------------------------- interval_seconds = (end - begin).total_seconds() if interval_seconds <= 0 or total_count == 0: return [] tz: tzinfo = begin.tzinfo # --- Step 1: Determine all calendar days touched by [begin, end) ---------- # # For example: # begin = Jan 1 12:00 # end = Jan 3 06:00 # # We need buckets for: # Jan 1 12:00 → Jan 2 00:00 # Jan 2 00:00 → Jan 3 00:00 # Jan 3 00:00 → Jan 3 06:00 # # Start at midnight on the day of `begin`. day_cursor = datetime(begin.year, begin.month, begin.day, tzinfo=tz) # If `begin` is earlier on that day (e.g. 10:00), we want that midnight. # If `begin` is past that midnight (e.g. 00:30), this is correct. # If `begin` is BEFORE that midnight (rare unless tz shifts), adjust: if day_cursor > begin: day_cursor -= timedelta(days=1) day_buckets: list[dict[str, Any]] = [] while day_cursor < end: day_start = day_cursor day_end = day_cursor + timedelta(days=1) # Overlap between [begin, end) and this day overlap_start = max(begin, day_start) overlap_end = min(end, day_end) overlap_seconds = (overlap_end - overlap_start).total_seconds() if overlap_seconds > 0: day_buckets.append( { "begin": overlap_start, "end": overlap_end, "seconds": overlap_seconds, } ) day_cursor = day_end # --- Step 2: Pro-rate counts across buckets ------------------------------- # # Compute the exact fractional count for each bucket: # bucket_fraction = bucket_seconds / interval_seconds # bucket_exact = total_count * bucket_fraction # # Then apply a "largest remainder" rounding strategy to ensure the sum # equals exactly total_count. exact_values: list[float] = [ (b["seconds"] / interval_seconds) * total_count for b in day_buckets ] floor_values: list[int] = [int(x) for x in exact_values] fractional_parts: list[float] = [x - int(x) for x in exact_values] # How many counts do we still need to distribute after flooring? remainder = total_count - sum(floor_values) # Sort buckets by descending fractional remainder indices_by_fraction = sorted( range(len(day_buckets)), key=lambda i: fractional_parts[i], reverse=True, ) # Start with floor values final_counts = floor_values[:] # Add +1 to the buckets with the largest fractional parts for idx in indices_by_fraction[:remainder]: final_counts[idx] += 1 # --- Step 3: Build the final per-day result list ------------------------- results: list[dict[str, Any]] = [] for bucket, count in zip(day_buckets, final_counts): if count > 0: results.append( { "begin": bucket["begin"], "end": bucket["end"], "count": count, } ) return results def _append_parsed_record( parsed_record: dict[str, Any], records: list[dict[str, Any]], begin_dt: datetime, end_dt: datetime, normalize: bool, ) -> None: """ Append a parsed DMARC record either unchanged or normalized. Args: parsed_record: The record returned by _parse_report_record(). records: Accumulating list of output records. begin_dt: Report-level begin datetime (UTC). end_dt: Report-level end datetime (UTC). normalize: Whether this report exceeded the allowed timespan and should be normalized per-day. """ if not normalize: parsed_record["normalized_timespan"] = False parsed_record["interval_begin"] = begin_dt.strftime("%Y-%m-%d %H:%M:%S") parsed_record["interval_end"] = end_dt.strftime("%Y-%m-%d %H:%M:%S") records.append(parsed_record) return # Normalization path: break record into daily buckets total_count = int(parsed_record.get("count", 0)) buckets = _bucket_interval_by_day(begin_dt, end_dt, total_count) if not buckets: return for part_index, bucket in enumerate(buckets): new_rec = parsed_record.copy() new_rec["count"] = bucket["count"] new_rec["normalized_timespan"] = True new_rec["interval_begin"] = bucket["begin"].strftime("%Y-%m-%d %H:%M:%S") new_rec["interval_end"] = bucket["end"].strftime("%Y-%m-%d %H:%M:%S") records.append(new_rec) def _scalar(value: Any) -> str | None: """Return the string an element the schema types as a string should hold. A string is returned as it is. A dict (an element with child elements, possibly beside text) gives its ``#text`` or ``None``; a list (a repeated element) or any other value gives ``None``. Without this a child element reached Elasticsearch and OpenSearch as an object and mapped the field as one (GHSA-qpcg-29vr-jwm3). """ if value is None or isinstance(value, str): return value if isinstance(value, dict): text = value.get("#text") return text if isinstance(text, str) else None return None def _scalar_fields(mapping: Any, keys: tuple[str, ...]) -> None: """Replace each of ``keys`` present in ``mapping`` with its :func:`_scalar` value, in place; a ``mapping`` that is not a dict is left to the caller's own error handling.""" if not isinstance(mapping, dict): return for key in keys: if key in mapping: mapping[key] = _scalar(mapping[key]) _REPORT_METADATA_SCALARS = ( "org_name", "email", "extra_contact_info", "report_id", "generator", ) _POLICY_PUBLISHED_SCALARS = ( "domain", "adkim", "aspf", "p", "sp", "pct", "fo", "np", "testing", "discovery_method", ) def _scalarize_report(report: dict[str, Any]) -> None: """Make the report-level fields the schema types as strings strings or ``None``, in place: ``version``, the ``report_metadata`` strings and every ``policy_published`` field of the RFC 7489 and RFC 9990 schemas (other children are not copied to the output). Records are handled by :func:`_parse_report_record`. Anything that is not a dict is left to the caller's own error handling.""" if not isinstance(report, dict): return _scalar_fields(report, ("version",)) _scalar_fields(report.get("report_metadata"), _REPORT_METADATA_SCALARS) policies = report.get("policy_published") for policy in policies if isinstance(policies, list) else [policies]: _scalar_fields(policy, _POLICY_PUBLISHED_SCALARS) def _strip_zone_id(value: str, field_name: str) -> str: """Return an IP address without its IPv6 zone ID, if it has one. A zone ID (``fe80::1%eth0``) is local to the reporting host, and the PostgreSQL ``inet`` type cannot hold one (GHSA-fxr7-7h9q-v949). ``%`` appears in no other form of an IP address, so everything from the first ``%`` is removed (``fe80::1%25eth0`` and ``fe80::1%`` also give ``fe80::1``), and a warning naming the field is logged. The remaining text is not checked here; the caller's own handling of an invalid IP address applies to it. Args: value: The address, as the report gives it field_name: The report field that gave it, for the warning Returns: str: ``value`` up to its first ``%`` """ address, sep, _zone = value.partition("%") if sep: logger.warning( f"Removed the IPv6 zone ID from {field_name}: {sanitize_for_log(value)}" ) return address @dataclass class _ReverseDNSBudget: """Per-report state that bounds the time an aggregate report spends in reverse DNS lookups (GHSA-rwfh-ww9c-jm9m). Attributes: seconds_left: Seconds of failed or slow lookups the report may still spend (``MAX_FAILED_REVERSE_DNS_SECONDS_PER_REPORT``) total_seconds_left: Seconds of lookups of any kind the report may still spend (``MAX_REVERSE_DNS_SECONDS_PER_REPORT``) memo: Source information already produced in this report, by lowercase IP address, so each distinct IP is looked up at most once per report exhausted: Whether either budget is spent; the report's remaining IPs are then enriched from the local IP database and reverse DNS map only """ seconds_left: float = field( default_factory=lambda: MAX_FAILED_REVERSE_DNS_SECONDS_PER_REPORT ) total_seconds_left: float = field( default_factory=lambda: MAX_REVERSE_DNS_SECONDS_PER_REPORT ) memo: dict[str, IPAddressInfo] = field(default_factory=dict) exhausted: bool = False _REVERSE_DNS_BUDGET_FALLBACK = ( "; the remaining records are enriched from the local IP database and " "reverse DNS map only" ) def _enrich_record_source( source_ip: str, config: ParserConfig, budget: _ReverseDNSBudget ) -> IPAddressInfo: """Return the source information for one record's ``source_ip``. A record's source IP is chosen by the report's sender, and a reverse zone that never answers costs the whole DNS lifetime (8 s with the default nameservers), so the time of every online lookup is charged to ``budget``. The time of a lookup that returns no PTR, or that takes at least ``SLOW_REVERSE_DNS_SECONDS`` even though it found one (the sender may control the reverse zone and answer every query just before the timeout), is charged to the failed-or-slow budget; the time of every lookup, however fast, is charged to the total budget as well, because a zone that answers just under the slow threshold is otherwise never charged and the number of records is not bounded. A lookup answered from the IP address cache is charged too, but takes microseconds. Once either budget is spent, one warning names the limit reached, and the remaining IPs are enriched from the local IP database and reverse DNS map only; those results are not cached, because the IP may well have a PTR. An IP already seen in this report is answered from ``budget.memo`` and not charged. """ ip = source_ip.lower() if ip in budget.memo: return budget.memo[ip] common: dict[str, Any] = dict( ip_db_path=config.ip_db_path, always_use_local_files=config.always_use_local_files, reverse_dns_map_path=config.reverse_dns_map_path, reverse_dns_map_url=config.reverse_dns_map_url, reverse_dns_map=config.reverse_dns_map, nameservers=config.nameservers, timeout=config.dns_timeout, retries=config.dns_retries, psl_overrides_path=config.psl_overrides_path, psl_overrides_url=config.psl_overrides_url, ) if budget.exhausted or config.offline: result = get_ip_address_info( ip, offline=True, cache=None if budget.exhausted else config.ip_address_cache, **common, ) else: start = time.monotonic() result = get_ip_address_info( ip, offline=False, cache=config.ip_address_cache, **common ) elapsed = time.monotonic() - start budget.total_seconds_left -= elapsed if result["reverse_dns"] is None or elapsed >= SLOW_REVERSE_DNS_SECONDS: budget.seconds_left -= elapsed if budget.seconds_left <= 0: budget.exhausted = True logger.warning( "Reverse DNS lookups for this report failed or were slow for " f"{MAX_FAILED_REVERSE_DNS_SECONDS_PER_REPORT:.0f} seconds" + _REVERSE_DNS_BUDGET_FALLBACK ) elif budget.total_seconds_left <= 0: budget.exhausted = True logger.warning( "Reverse DNS lookups for this report took more than " f"{MAX_REVERSE_DNS_SECONDS_PER_REPORT:.0f} seconds" + _REVERSE_DNS_BUDGET_FALLBACK ) budget.memo[ip] = result return result def _parse_report_record( record: dict[str, Any], *, config: ParserConfig, is_rfc_9990: bool = False, dns_budget: _ReverseDNSBudget | None = None, ) -> dict[str, Any]: """ Converts a record from a DMARC aggregate report into a more consistent format Args: record (dict): The record to convert config (ParserConfig): Parsing and enrichment options, plus caches is_rfc_9990 (bool): Whether the enclosing report was detected as RFC 9990-shaped, for RFC 9990-aware validation warnings dns_budget (_ReverseDNSBudget | None): The enclosing report's reverse DNS budget; ``None`` gives the record a budget of its own Returns: dict: The converted record Raises: ValueError: If the source IP is empty or not an IP address (after its IPv6 zone ID, if any, is removed), or the count is not an integer between 0 and ``MAX_MESSAGE_COUNT`` """ record = record.copy() new_record: dict[str, Any] = {} # Fields the schema types as strings become strings or None # (GHSA-qpcg-29vr-jwm3); see _scalar(). _scalar_fields(record["row"], ("source_ip",)) if record["row"]["source_ip"] is None: raise ValueError("Source IP address is empty") source_ip = _strip_zone_id(record["row"]["source_ip"], "source_ip") # A <count> with no text of its own (child elements only, or repeated) # becomes "", so int() raises the ValueError of a non-numeric count. count = int(_scalar(record["row"]["count"]) or "") # A count no output's 32-bit integer column can hold is rejected here, # not by the output (GHSA-fxr7-7h9q-v949), and before the reverse DNS # lookup, which a record that is dropped anyway should not wait on. if not 0 <= count <= MAX_MESSAGE_COUNT: raise ValueError(f"count {count} is outside 0..{MAX_MESSAGE_COUNT}") if dns_budget is None: dns_budget = _ReverseDNSBudget() new_record["source"] = _enrich_record_source(source_ip, config, dns_budget) new_record["count"] = count policy_evaluated = record["row"]["policy_evaluated"].copy() _scalar_fields(policy_evaluated, ("disposition", "dkim", "spf")) new_policy_evaluated: dict[str, Any] = { "disposition": "none", "dkim": "fail", "spf": "fail", "policy_override_reasons": [], } if "disposition" in policy_evaluated: new_policy_evaluated["disposition"] = _normalize_result_word( policy_evaluated["disposition"] ) for key in ("dkim", "spf"): if key in policy_evaluated: new_policy_evaluated[key] = _normalize_result_word(policy_evaluated[key]) reasons = [] spf_aligned = ( policy_evaluated["spf"] is not None and policy_evaluated["spf"].lower() == "pass" ) dkim_aligned = ( policy_evaluated["dkim"] is not None and policy_evaluated["dkim"].lower() == "pass" ) dmarc_aligned = spf_aligned or dkim_aligned new_record["alignment"] = dict() new_record["alignment"]["spf"] = spf_aligned new_record["alignment"]["dkim"] = dkim_aligned new_record["alignment"]["dmarc"] = dmarc_aligned if "reason" in policy_evaluated: if type(policy_evaluated["reason"]) is list: reasons = policy_evaluated["reason"] else: reasons = [policy_evaluated["reason"]] normalized_reasons = [] for reason in reasons: # `comment` and `type` are strings in both schemas, but a sender can # give them child elements (a dict) or repeat them (a list), and the # CSV writer joins them as strings (GHSA-4272-4r8g-6c5h). They follow # the rule of every other string field: see _scalar(). reason_type = _scalar(reason.get("type")) normalized_reasons.append( {"type": reason_type, "comment": _scalar(reason.get("comment"))} ) if is_rfc_9990 and reason_type in RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES: logger.warning( "Policy override reason type %r was removed in RFC 9990; " "expected one of %s", reason_type, sorted(RFC_9990_POLICY_OVERRIDE_TYPES), ) elif ( is_rfc_9990 and reason_type is not None and reason_type not in RFC_9990_POLICY_OVERRIDE_TYPES ): logger.warning( "Unknown policy override reason type %r per RFC 9990; " "expected one of %s", reason_type, sorted(RFC_9990_POLICY_OVERRIDE_TYPES), ) new_policy_evaluated["policy_override_reasons"] = normalized_reasons new_record["policy_evaluated"] = new_policy_evaluated raw_identifiers = ( record["identities"] if "identities" in record else record["identifiers"] ) new_record["identifiers"] = { key: _scalar(value) for key, value in raw_identifiers.items() if key in ("header_from", "envelope_from", "envelope_to") } new_record["auth_results"] = {"dkim": [], "spf": []} if type(new_record["identifiers"]["header_from"]) is str: lowered_from = new_record["identifiers"]["header_from"].lower() else: lowered_from = "" new_record["identifiers"]["header_from"] = lowered_from if isinstance(record["auth_results"], dict): auth_results = record["auth_results"].copy() if "spf" not in auth_results: auth_results["spf"] = [] if "dkim" not in auth_results: auth_results["dkim"] = [] else: auth_results = new_record["auth_results"].copy() if not isinstance(auth_results["dkim"], list): auth_results["dkim"] = [auth_results["dkim"]] for result in auth_results["dkim"]: _scalar_fields(result, ("domain", "selector", "result", "human_result")) if "domain" in result and result["domain"] is not None: new_result: dict[str, Any] = {"domain": result["domain"]} if "selector" in result and result["selector"] is not None: new_result["selector"] = result["selector"] else: if is_rfc_9990: logger.warning( "DKIM auth result for %r is missing the 'selector' " "element, which is REQUIRED by RFC 9990", result["domain"], ) new_result["selector"] = "none" if "result" in result and result["result"] is not None: new_result["result"] = _normalize_result_word(result["result"]) else: new_result["result"] = "none" new_result["human_result"] = _text(result.get("human_result")) new_record["auth_results"]["dkim"].append(new_result) if not isinstance(auth_results["spf"], list): auth_results["spf"] = [auth_results["spf"]] for result in auth_results["spf"]: _scalar_fields(result, ("domain", "scope", "result", "human_result")) if "domain" in result and result["domain"] is not None: new_result: dict[str, Any] = {"domain": result["domain"]} if "scope" in result and result["scope"] is not None: new_result["scope"] = result["scope"] else: new_result["scope"] = "mfrom" if "result" in result and result["result"] is not None: new_result["result"] = _normalize_result_word(result["result"]) else: new_result["result"] = "none" new_result["human_result"] = _text(result.get("human_result")) new_record["auth_results"]["spf"].append(new_result) # Backfill envelope_from from the last SPF result's domain when the # reporter omitted the identifier or sent it empty. if new_record["identifiers"].get("envelope_from") is None: envelope_from = None if len(auth_results["spf"]) > 0: spf_result = auth_results["spf"][-1] if "domain" in spf_result: envelope_from = spf_result["domain"] if envelope_from is not None: envelope_from = str(envelope_from).lower() new_record["identifiers"]["envelope_from"] = envelope_from envelope_to = None if "envelope_to" in new_record["identifiers"]: envelope_to = new_record["identifiers"]["envelope_to"] del new_record["identifiers"]["envelope_to"] new_record["identifiers"]["envelope_to"] = envelope_to return new_record def _non_blank(value: Any) -> Any: """Treat ``None`` or a blank/whitespace-only string as absent. Some SMTP TLS (RFC 8460) reporters send an empty string for an optional failure-details field (e.g. ``sending-mta-ip``/``receiving-ip`` on a ``sts-policy-fetch-error``) instead of omitting the key. An empty string is not a valid IP address, so passing it through crashed the PostgreSQL save (``INET`` columns) and would also have crashed the Elasticsearch and OpenSearch saves: their ``Ip()`` fields' validation (``elasticsearch.dsl``/``opensearchpy``'s ``Document.save()``) rejects an empty string with a bare ``ValueError`` (issue #915). Non-string, non-blank values pass through unchanged. """ if value is None: return None if isinstance(value, str) and value.strip() == "": return None return value # RFC 8460 §4.4 gives every SMTP TLS report date as an RFC 3339 date-time, # which is at most about 35 characters even with a long fractional second. # dateutil's parser is super-linear on a long run of digits (12.4 s for # 1,000,000 of them, Python 3.14), so a longer value is rejected unparsed. _SMTP_TLS_MAX_DATETIME_LENGTH = 64 def _smtp_tls_string(value: Any, field: str) -> str: """Return ``value`` if it is a string, as RFC 8460 §4.4 requires of ``field``; raise ``InvalidSMTPTLSReport`` otherwise.""" if not isinstance(value, str): raise InvalidSMTPTLSReport( f"{field} must be a string, not {type(value).__name__}" ) _smtp_tls_check_utf8(value, field) return value def _smtp_tls_check_utf8(value: str, field: str) -> None: """Raise ``InvalidSMTPTLSReport`` when ``value`` holds a lone surrogate (JSON ``"\\ud800"``). RFC 8460 §4 reports are I-JSON, whose RFC 7493 §2.1 forbids surrogate code points, and the outputs cannot encode one.""" try: value.encode("utf-8") except UnicodeEncodeError as error: raise InvalidSMTPTLSReport(f"{field} is not valid UTF-8 text") from error def _smtp_tls_string_list(value: Any, field: str) -> list[str]: """Return ``value`` if it is a list of strings (RFC 8460 §4.4's "JSON array of strings"); raise ``InvalidSMTPTLSReport`` otherwise.""" if not isinstance(value, list) or not all(isinstance(item, str) for item in value): raise InvalidSMTPTLSReport(f"{field} must be a list of strings") for item in value: _smtp_tls_check_utf8(item, field) return value def _smtp_tls_mx_host_patterns(value: Any, field: str) -> list[str]: """Return a policy's MX host patterns as a list of strings. RFC 8460 §4.4 describes the value as "a JSON array of strings", but its Appendix B example gives one pattern as a plain string, so a string becomes a one-element list. ``null`` and an empty string give an empty list; any other type raises ``InvalidSMTPTLSReport``. """ if value is None or value == "": return [] if isinstance(value, str): return [_smtp_tls_string(value, field)] return _smtp_tls_string_list(value, field) def _smtp_tls_count(value: Any, field: str) -> int | None: """Return ``value`` if it is null or an integer from 0 to ``MAX_MESSAGE_COUNT``. RFC 8460 §4.4 encodes each session count as "an integer, encoded as a JSON number". ``null`` is tolerated as an unreported count. A JSON ``true``/``false`` is rejected even though ``bool`` subclasses ``int``. A count outside ``0..MAX_MESSAGE_COUNT`` is rejected because the PostgreSQL output stores counts as 32-bit integers (GHSA-fxr7-7h9q-v949). Raises ``InvalidSMTPTLSReport`` otherwise. """ if value is None: return value if not isinstance(value, int) or isinstance(value, bool): raise InvalidSMTPTLSReport( f"{field} must be an integer, not {type(value).__name__}" ) if not 0 <= value <= MAX_MESSAGE_COUNT: raise InvalidSMTPTLSReport(f"{field} must be between 0 and {MAX_MESSAGE_COUNT}") return value def _smtp_tls_datetime(value: Any, field: str) -> str: """Return ``value`` if it is a date-time string every output can convert. RFC 8460 §4.4 gives ``start-datetime`` and ``end-datetime`` as strings in RFC 3339 format. The value is checked with the conversions the Elasticsearch, OpenSearch and S3 outputs (a UTC ``datetime``) and the Splunk output (a UNIX timestamp) make later. dateutil also accepts a partial date such as ``"2016"``, which PostgreSQL may still reject; that failure is raised as ``PostgreSQLRejectedReport``, which skips the report in the PostgreSQL output only (an output error with ``fail_on_output_error``). A ``start-datetime`` more than ``MAX_REPORT_FUTURE_DAYS`` days in the future is also rejected, because it names the dated index (#947); ``end-datetime`` is not checked. Raises ``InvalidSMTPTLSReport`` otherwise. """ value = _smtp_tls_string(value, field) if len(value) > _SMTP_TLS_MAX_DATETIME_LENGTH: raise InvalidSMTPTLSReport( f"{field} is longer than {_SMTP_TLS_MAX_DATETIME_LENGTH} characters" ) try: moment = human_timestamp_to_datetime(value, to_utc=True) human_timestamp_to_unix_timestamp(value) except (ValueError, OverflowError) as e: raise InvalidSMTPTLSReport( f"{field} is not a valid date-time: {sanitize_for_log(value)}" ) from e if field == "start-datetime": try: _check_not_future(moment, field) except ValueError as error: raise InvalidSMTPTLSReport(str(error)) from error return value def _parse_smtp_tls_failure_details(failure_details: dict[str, Any]): try: new_failure_details: dict[str, Any] = { "result_type": _smtp_tls_string( failure_details["result-type"], "result-type" ), "failed_session_count": _smtp_tls_count( failure_details["failed-session-count"], "failed-session-count" ), } optional_fields = [ ("sending-mta-ip", "sending_mta_ip"), ("receiving-ip", "receiving_ip"), ("receiving-mx-hostname", "receiving_mx_hostname"), ("receiving-mx-helo", "receiving_mx_helo"), ("failure-reason-code", "failure_reason_code"), ] for src_key, dest_key in optional_fields: value = _non_blank(failure_details.get(src_key)) if value is not None: new_failure_details[dest_key] = _smtp_tls_string(value, src_key) # RFC 8460 §4.4 defines both as the IP address of the sending or # receiving MTA, and both are optional. Drop a value that is not an # IP address, which the PostgreSQL inet column and the # Elasticsearch/OpenSearch ip mappings reject (GHSA-fxr7-7h9q-v949). # An IPv6 zone ID (fe80::1%eth0) is local to the reporting host and # the inet type cannot hold one, and ipaddress.ip_address() accepts # it, so such a value is dropped as well. The aggregate and failure # paths strip the zone ID instead (_strip_zone_id()), because the # source IP is required there; these fields are optional, so # dropping one loses nothing a report needs. for src_key, dest_key in ( ("sending-mta-ip", "sending_mta_ip"), ("receiving-ip", "receiving_ip"), ): value = new_failure_details.get(dest_key) if value is None: continue try: if "%" in value: raise ValueError("zone ID") ipaddress.ip_address(value) except ValueError: logger.warning( f"Ignoring {src_key} that is not an IP address: " f"{sanitize_for_log(value)}" ) del new_failure_details[dest_key] # RFC 8460 §4.4 names the JSON key "additional-information" (its # value is described as an "additional-info-uri"). Some reporters # instead send the non-RFC key "additional-info-uri" directly; fall # back to it for backward compatibility when the RFC key is absent # or blank. additional_info_uri = _non_blank(failure_details.get("additional-information")) if additional_info_uri is None: additional_info_uri = _non_blank(failure_details.get("additional-info-uri")) if additional_info_uri is not None: new_failure_details["additional_info_uri"] = _smtp_tls_string( additional_info_uri, "additional-information" ) return new_failure_details except KeyError as e: raise InvalidSMTPTLSReport( f"Missing required failure details field: {e}" ) from e except Exception as e: raise InvalidSMTPTLSReport(str(e) + _exc_origin(e)) from e def _parse_smtp_tls_report_policy(policy: dict[str, Any]): policy_types = ["tlsa", "sts", "no-policy-found"] try: policy_domain = _smtp_tls_string( policy["policy"]["policy-domain"], "policy-domain" ) policy_type = _smtp_tls_string(policy["policy"]["policy-type"], "policy-type") failure_details = [] if policy_type not in policy_types: raise InvalidSMTPTLSReport( f"Invalid policy type {sanitize_for_log(str(policy_type))}" ) new_policy: dict[str, Any] = { "policy_domain": policy_domain, "policy_type": policy_type, } if policy["policy"].get("policy-string") is not None: policy_strings = _smtp_tls_string_list( policy["policy"]["policy-string"], "policy-string" ) if len(policy_strings) > 0: new_policy["policy_strings"] = policy_strings # RFC 8460 §4.4 keys the policy's MX host patterns "mx-host"; # "mx-host-pattern" names its value type, not a key, but was read as # the key before and is kept as a fallback (issue #942). mx_host_patterns = _smtp_tls_mx_host_patterns( policy["policy"].get("mx-host"), "mx-host" ) if not mx_host_patterns: mx_host_patterns = _smtp_tls_mx_host_patterns( policy["policy"].get("mx-host-pattern"), "mx-host-pattern" ) if mx_host_patterns: new_policy["mx_host_patterns"] = mx_host_patterns new_policy["successful_session_count"] = _smtp_tls_count( policy["summary"]["total-successful-session-count"], "total-successful-session-count", ) new_policy["failed_session_count"] = _smtp_tls_count( policy["summary"]["total-failure-session-count"], "total-failure-session-count", ) if "failure-details" in policy: if not isinstance(policy["failure-details"], list) or not all( isinstance(details, dict) for details in policy["failure-details"] ): raise InvalidSMTPTLSReport("failure-details must be a list of objects") for details in policy["failure-details"]: failure_details.append(_parse_smtp_tls_failure_details(details)) new_policy["failure_details"] = failure_details return new_policy except KeyError as e: raise InvalidSMTPTLSReport(f"Missing required policy field: {e}") from e except Exception as e: raise InvalidSMTPTLSReport(str(e) + _exc_origin(e)) from e
[docs] def parse_smtp_tls_report_json(report: str | bytes) -> SMTPTLSReport: """Parses and validates an SMTP TLS report""" required_fields = [ "organization-name", "date-range", "contact-info", "report-id", "policies", ] if _exceeds_size_cap(report): # json.loads() holds the whole document at several times its size # (GHSA-q7gm-8cj9-94hr). raise InvalidSMTPTLSReport(_size_cap_text(report)) try: if isinstance(report, bytes): report = report.decode("utf-8", errors="replace") policies = [] report_dict = json.loads(report) for required_field in required_fields: if required_field not in report_dict: raise Exception(f"Missing required field: {required_field}]") if not isinstance(report_dict["policies"], list): policies_type = type(report_dict["policies"]) raise InvalidSMTPTLSReport(f"policies must be a list, not {policies_type}") for policy in report_dict["policies"]: policies.append(_parse_smtp_tls_report_policy(policy)) # RFC 8460 §4.4 gives contact-info as one string; some reporters # send a list of them, which is kept as it is. contact_info = report_dict["contact-info"] if isinstance(contact_info, str): contact_info = _smtp_tls_string(contact_info, "contact-info") else: contact_info = _smtp_tls_string_list(contact_info, "contact-info") new_report: SMTPTLSReport = { "organization_name": _smtp_tls_string( report_dict["organization-name"], "organization-name" ), "begin_date": _smtp_tls_datetime( report_dict["date-range"]["start-datetime"], "start-datetime" ), "end_date": _smtp_tls_datetime( report_dict["date-range"]["end-datetime"], "end-datetime" ), "contact_info": contact_info, "report_id": _smtp_tls_string(report_dict["report-id"], "report-id"), "policies": policies, } return new_report except KeyError as e: raise InvalidSMTPTLSReport(f"Missing required field: {e}") from e except Exception as e: raise InvalidSMTPTLSReport(str(e) + _exc_origin(e)) from e
# Characters that make a spreadsheet application read a cell as a formula. _CSV_FORMULA_PREFIXES = ("=", "+", "-", "@", "\t", "\r") def _csv_safe(value: Any) -> Any: """Makes a string CSV field safe to write and to open in a spreadsheet. Two things happen to a string, in this order: 1. NUL characters are stripped. Python 3.10's ``csv`` writer raised ``_csv.Error: need to escape, but no escapechar set`` on any field containing NUL (CPython issue 97503, fixed in 3.11; the floor is now 3.11), and NUL is still dropped to match ``get_filename_safe_string``. 2. A string starting with ``=``, ``+``, ``-``, ``@``, tab, or carriage return gets a leading ``'``. Spreadsheet applications evaluate such a cell as a formula, and report text fields come from untrusted mail, so a reporter could otherwise plant a formula like ``=HYPERLINK(...)`` in the output (CWE-1236). The apostrophe is the OWASP CSV Injection recommendation, and makes the application treat the cell as text: https://owasp.org/www-community/attacks/CSV_Injection Non-string values (ASN integers, counts) are returned unchanged. """ if isinstance(value, str): value = value.replace("\x00", "") if value.startswith(_CSV_FORMULA_PREFIXES): return "'" + value return value return value
[docs] def parsed_smtp_tls_reports_to_csv_rows( reports: SMTPTLSReport | list[SMTPTLSReport], ) -> list[dict[str, Any]]: """Converts one or more parsed SMTP TLS reports into a list of single-layer dict objects suitable for use in a CSV""" if isinstance(reports, dict): reports = [reports] rows = [] for report in reports: common_fields: dict[str, Any] = { "organization_name": report["organization_name"], "begin_date": report["begin_date"], "end_date": report["end_date"], "report_id": report["report_id"], } record: dict[str, Any] = common_fields.copy() for policy in report["policies"]: if "policy_strings" in policy: record["policy_strings"] = "|".join(policy["policy_strings"]) if "mx_host_patterns" in policy: record["mx_host_patterns"] = "|".join(policy["mx_host_patterns"]) successful_record = record.copy() successful_record["policy_domain"] = policy["policy_domain"] successful_record["policy_type"] = policy["policy_type"] successful_record["successful_session_count"] = policy[ "successful_session_count" ] successful_record["failed_session_count"] = policy["failed_session_count"] rows.append(successful_record) if "failure_details" in policy: for failure_details in policy["failure_details"]: failure_record = record.copy() for key in failure_details.keys(): failure_record[key] = failure_details[key] rows.append(failure_record) return rows
[docs] def parsed_smtp_tls_reports_to_csv( reports: SMTPTLSReport | list[SMTPTLSReport], ) -> str: """ Converts one or more parsed SMTP TLS reports to flat CSV format, including headers Args: reports: A parsed SMTP TLS report or list of parsed SMTP TLS reports Returns: str: Parsed SMTP TLS report data in flat CSV format, including headers """ fields = [ "organization_name", "begin_date", "end_date", "report_id", "result_type", "successful_session_count", "failed_session_count", "policy_domain", "policy_type", "policy_strings", "mx_host_patterns", "sending_mta_ip", "receiving_ip", "receiving_mx_hostname", "receiving_mx_helo", "additional_info_uri", "failure_reason_code", ] csv_file_object = StringIO(newline="\n") writer = DictWriter(csv_file_object, fields) writer.writeheader() rows = parsed_smtp_tls_reports_to_csv_rows(reports) for row in rows: writer.writerow({key: _csv_safe(value) for key, value in row.items()}) csv_file_object.flush() return csv_file_object.getvalue()
# No field of an aggregate report carries an attribute parsedmarc keeps, and # an attribute xmltodict kept (<p a="1">none</p> as {"@a": "1", "#text": # "none"}) would reach Elasticsearch or OpenSearch as an object and map the # field as one (GHSA-qpcg-29vr-jwm3). With xml_attribs=False xmltodict builds # no attribute keys, while expat still hands the attributes to the start # handler, so _LimitedExpatParser counts them. def _parse_limited_xml(xml: str) -> Any: """Parse ``xml`` with xmltodict through the limited expat parser, without attributes, and raise ``InvalidAggregateReport`` when it goes over ``MAX_XML_DEPTH`` or ``MAX_XML_NODES``.""" try: return xmltodict.parse(xml, expat=_NoDoctypeExpat, xml_attribs=False) except _XMLLimitExceeded as error: raise InvalidAggregateReport(f"Invalid XML: {error}") from error def _recover_xml(xml: str) -> str: """Return ``xml`` repaired by lxml's recovery parser, for a document expat rejected: ``""`` when nothing is recoverable, ``"<a/>"`` when the recovery itself fails. Raises ``InvalidAggregateReport`` for a document over the node limit or holding a DOCTYPE.""" has_doctype = False try: tree = _parse_xml_with_recovery(xml.encode("utf-8")) root = tree.getroot() if root is None: # Nothing recoverable; the second parse reports it as # invalid XML. return "" # The first parse can fail before expat reaches a DOCTYPE (e.g. on # a malformed XML declaration, which is only rewritten later), so # the recovered tree gets the same DOCTYPE check, made before # serializing. The rejection is raised after this try, which would # otherwise swallow it. has_doctype = bool(tree.docinfo.doctype) or ( tree.docinfo.internalDTD is not None ) if not has_doctype: # Serializing the root element rather than the tree leaves any # internal DTD subset out of the XML handed to the second parse. # str() is a no-op at runtime; lxml ships no type stubs, so # pyright would widen the result to str | bytes. return str(etree.tostring(root, encoding="unicode")) except _XMLLimitExceeded as limit: raise InvalidAggregateReport(f"Invalid XML: {limit}") from limit except Exception: return "<a/>" raise InvalidAggregateReport(_DOCTYPE_ERROR) def _size_cap_text(content: str | bytes) -> str: """Return the reason a report over the size cap is rejected: bytes are measured as they are, text as UTF-8.""" size = f"{MAX_DECOMPRESSED_REPORT_SIZE // 2**20} MiB" if isinstance(content, bytes): return f"the report is larger than {size}" return f"the report is larger than {size} of UTF-8" def _exceeds_size_cap(content: str | bytes) -> bool: """Return whether ``content`` is more than ``MAX_DECOMPRESSED_REPORT_SIZE`` bytes, a ``str`` measured as UTF-8, as the cap on a compressed report measures its decompressed bytes. ASCII text and text too long or too short to matter are decided by length; other text is encoded 1 MiB of characters at a time, so the measure needs no full copy.""" cap = MAX_DECOMPRESSED_REPORT_SIZE if isinstance(content, bytes) or len(content) > cap: return len(content) > cap # A character is at most 4 bytes of UTF-8. if len(content) <= cap // 4 or content.isascii(): return False size = 0 for offset in range(0, len(content), 2**20): size += len(content[offset : offset + 2**20].encode("utf-8", "surrogatepass")) if size > cap: return True return False def _report_text(xml: str | bytes) -> str: """Return the report as text, after :func:`_check_xml_text`; bytes over the size cap are rejected before they are decoded (errors ignored).""" if isinstance(xml, bytes): if _exceeds_size_cap(xml): raise InvalidAggregateReport(f"Invalid XML: {_size_cap_text(xml)}") xml = xml.decode(errors="ignore") _check_xml_text(xml) return xml def _check_xml_text(xml: str) -> None: """Reject, before any parser reads it, an aggregate report whose text alone would make parsing it too costly (GHSA-q7gm-8cj9-94hr): more than ``MAX_DECOMPRESSED_REPORT_SIZE`` bytes of UTF-8 (an uncompressed attachment or file is held to the cap a compressed one is), more attributes than ``MAX_XML_NODES``, or a tag longer than ``MAX_XML_TAG_LENGTH``. Raises ``InvalidAggregateReport``.""" if _exceeds_size_cap(xml): raise InvalidAggregateReport(f"Invalid XML: {_size_cap_text(xml)}") # The = count is one C pass, so it runs before the slower tag scan. _check_attribute_count(xml) _check_tag_lengths(xml) def _check_tag_lengths(xml: str) -> None: """Raise ``InvalidAggregateReport`` when any text between a ``<`` and the next ``>`` is longer than ``MAX_XML_TAG_LENGTH``, before a parser builds the tag's attribute list (GHSA-q7gm-8cj9-94hr). One linear pass without a regex; the text after the last ``>`` is checked too, so an unterminated tag is caught. A comment or CDATA section is treated as a tag, which is harmless at this length. XML allows a raw ``>`` inside a quoted attribute value, which ends a piece early, so this scan does not bound how many attributes a tag holds; :func:`_check_attribute_count` does. """ # The text is split on ">" one chunk at a time: one split of the whole # text made a string per tag, +1,318 MiB for 100 MiB of "<a/>". A # chunk's pieces are measured with max() before any is examined, and # the unfinished last piece is carried into the next chunk from its # last "<", so memory stays near one chunk. carry = "" for offset in range(0, len(xml), _TAG_SCAN_CHUNK): pieces = (carry + xml[offset : offset + _TAG_SCAN_CHUNK]).split(">") carry = pieces.pop() if max(map(len, pieces), default=0) > MAX_XML_TAG_LENGTH: for piece in pieces: _check_tag_piece(piece) carry = _check_tag_piece(carry) def _check_attribute_count(xml: str) -> None: """Raise ``InvalidAggregateReport`` when ``xml`` holds more ``=`` characters than ``MAX_XML_NODES``: every attribute needs one, and expat and libxml2 build a tag's attribute list before the node counter sees it (GHSA-q7gm-8cj9-94hr). ``str.count`` is one linear pass.""" if xml.count("=") > MAX_XML_NODES: raise InvalidAggregateReport( f"Invalid XML: more than {MAX_XML_NODES:,} `=` characters, so more " "attributes than the node limit allows" ) def _check_tag_piece(piece: str) -> str: """Raise the tag-length error when the text of ``piece`` from its last ``<`` is longer than ``MAX_XML_TAG_LENGTH``; return that text, or ``""`` when ``piece`` holds no ``<``.""" start = piece.rfind("<") if start < 0: return "" if len(piece) - start > MAX_XML_TAG_LENGTH: raise InvalidAggregateReport( f"Invalid XML: a tag is longer than {MAX_XML_TAG_LENGTH:,} characters" ) return piece[start:] _TAG_SCAN_CHUNK = 1 << 16 def _check_report_span(span_seconds: int) -> None: """Raise ``InvalidAggregateReport`` when a report's date range is negative or longer than ``MAX_REPORT_SPAN_DAYS``, before normalization splits it into one record per day (GHSA-p85v-578x-3wcf). The message holds no number from the report, whose dates can be thousands of digits long.""" if span_seconds < 0: raise InvalidAggregateReport("Report date range is negative") if span_seconds > MAX_REPORT_SPAN_DAYS * 86400: raise InvalidAggregateReport( f"Report date range spans more than {MAX_REPORT_SPAN_DAYS} days" ) def _check_not_future(moment: datetime, what: str) -> None: """Raise ``ValueError`` when ``moment`` is more than ``MAX_REPORT_FUTURE_DAYS`` days ahead of the current time, because the date that names a report's dated index would then create an index for a date far in the future (#947). ``moment`` must be an aware UTC datetime; each parser converts the error to its own ``Invalid*Report``.""" if moment > datetime.now(timezone.utc) + timedelta(days=MAX_REPORT_FUTURE_DAYS): raise ValueError( f"{what} {moment.isoformat()} is more than " f"{MAX_REPORT_FUTURE_DAYS} days in the future" ) def _days_touched(begin: datetime, end: datetime) -> int: """Return how many calendar days of ``begin``'s timezone the interval [begin, end) touches: the number of buckets ``_bucket_interval_by_day()`` makes, before it drops the buckets that receive no messages.""" day_start = datetime(begin.year, begin.month, begin.day, tzinfo=begin.tzinfo) return max(0, -(-int((end - day_start).total_seconds()) // 86400)) def _normalize_within_limit( normalize: bool, xml_records: Any, begin_dt: datetime, end_dt: datetime, report_id: str, ) -> bool: """Return whether per-day normalization stays enabled for a report. Normalization makes one record per record and day, so a report whose records times days exceed ``MAX_NORMALIZED_RECORDS`` keeps its records whole, over the report's own range, and one warning is logged (GHSA-p85v-578x-3wcf). """ if not normalize: return False record_count = len(xml_records) if isinstance(xml_records, list) else 1 days = _days_touched(begin_dt, end_dt) if record_count * days <= MAX_NORMALIZED_RECORDS: return True logger.warning( f"Report {sanitize_for_log(str(report_id))} is not normalized to daily " f"records: {record_count} records over {days} days would make " f"{record_count * days} records, more than {MAX_NORMALIZED_RECORDS}" ) return False def _parse_report_records( xml_records: Any, *, config: ParserConfig, is_rfc_9990: bool, begin_dt: datetime, end_dt: datetime, normalize: bool, keep_alive: Callable | None, dns_budget: _ReverseDNSBudget | None = None, ) -> list[dict[str, Any]]: """Parse an aggregate report's ``<record>`` elements (a list, or one record) into output records, split per day when ``normalize`` is set. A record of a list that fails to parse is skipped with a warning. ``dns_budget`` is shared by every record (``None`` creates one); see :class:`_ReverseDNSBudget`.""" if dns_budget is None: dns_budget = _ReverseDNSBudget() records: list[dict[str, Any]] = [] if type(xml_records) is list: for i in range(len(xml_records)): if keep_alive is not None and i > 0 and i % 20 == 0: logger.debug("Sending keepalive cmd") keep_alive() logger.debug(f"Processed {i}/{len(xml_records)}") try: report_record = _parse_report_record( xml_records[i], config=config, is_rfc_9990=is_rfc_9990, dns_budget=dns_budget, ) _append_parsed_record( parsed_record=report_record, records=records, begin_dt=begin_dt, end_dt=end_dt, normalize=normalize, ) except Exception as e: logger.warning(f"Could not parse record: {sanitize_for_log(str(e))}") else: report_record = _parse_report_record( xml_records, config=config, is_rfc_9990=is_rfc_9990, dns_budget=dns_budget, ) _append_parsed_record( parsed_record=report_record, records=records, begin_dt=begin_dt, end_dt=end_dt, normalize=normalize, ) return records
[docs] def parse_aggregate_report_xml( xml: str | bytes, *, ip_db_path: str | None = None, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, offline: bool = False, nameservers: list[str] | None = None, timeout: float = DEFAULT_DNS_TIMEOUT, retries: int = DEFAULT_DNS_MAX_RETRIES, keep_alive: Callable | None = None, normalize_timespan_threshold_hours: float = 24.0, config: ParserConfig | None = None, ) -> AggregateReport: """Parses a DMARC XML report string and returns a consistent dict Args: xml (str | bytes): DMARC aggregate report XML (bytes are decoded with errors ignored) ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP always_use_local_files (bool): Do not download files reverse_dns_map_path (str): Path to a reverse DNS map file reverse_dns_map_url (str): URL to a reverse DNS map file offline (bool): Do not query online for geolocation or DNS nameservers (list): A list of one or more nameservers to use (Cloudflare's public DNS resolvers by default) timeout (float): Sets the DNS timeout in seconds retries (int): Number of times to retry DNS queries on timeout or other transient errors keep_alive (callable): Keep alive function. Not part of ``config``; always applies. normalize_timespan_threshold_hours (float): Normalize timespans beyond this config (ParserConfig): a single object carrying all parsing and enrichment options plus the caches; when provided, the individual option keyword arguments listed above are ignored in favor of the config's values. Returns: dict: The parsed aggregate DMARC report """ cfg = _resolve_config( config, offline=offline, ip_db_path=ip_db_path, always_use_local_files=always_use_local_files, reverse_dns_map_path=reverse_dns_map_path, reverse_dns_map_url=reverse_dns_map_url, nameservers=nameservers, dns_timeout=timeout, dns_retries=retries, normalize_timespan_threshold_hours=normalize_timespan_threshold_hours, ) errors = [] # Parse XML and recover from errors xml = _report_text(xml) # Detect the XML namespace before any rewriting strips it. The dmarc-2.0 # namespace is one of the indicators for an RFC 9990 report but it is # NOT a reliable sole discriminator: the <version> element value is # ambiguous (RFC 9990's appendix sample uses <version>1.0</version> # inside the dmarc-2.0 namespace), and real-world reporters frequently # emit RFC 9990-shaped reports without declaring the namespace at all. # The final `is_rfc_9990` decision is made post-parse so that # RFC 9990-only fields (np, testing, discovery_method, generator, # human_result) can also vote it in. xml_namespace: str | None = None namespace_match = xml_namespace_regex.search(xml) if namespace_match: xml_namespace = namespace_match.group(1) first_error: str | None = None try: xmltodict.parse(xml, expat=_NoDoctypeExpat, xml_attribs=False)["feedback"] except _XMLDoctypeFound as e: raise InvalidAggregateReport(_DOCTYPE_ERROR) from e except _XMLLimitExceeded as e: # Rejected outright: the lxml recovery below would be a second pass # over the same oversized document. raise InvalidAggregateReport(f"Invalid XML: {e}") from e except Exception as e: first_error = f"Invalid XML: {e.__str__()}" # The recovery runs after the except block, not inside it: while a # handler runs, the handled exception's traceback keeps the failed # parse's frames, and with them its partial tree, alive. if first_error is not None: errors.append(first_error) xml = _recover_xml(xml) try: # Replace XML header (sometimes they are invalid) xml = xml_header_regex.sub('<?xml version="1.0"?>', xml) # Remove invalid schema tags xml = xml_schema_regex.sub("", xml) report = _parse_limited_xml(xml)["feedback"] _scalarize_report(report) report_metadata = report["report_metadata"] # The report_metadata strings are strings or None here: # xmltodict kept no xml:lang attribute (xml_attribs=False) and # _scalarize_report() any child element. Text in place of the # section's elements is a missing section. if not isinstance(report_metadata, dict): raise AttributeError("report_metadata holds no elements") schema = "draft" if "version" in report: schema = report["version"] new_report: dict[str, Any] = { "xml_schema": schema, "xml_namespace": xml_namespace, } new_report_metadata: dict[str, Any] = {} if report_metadata["org_name"] is None: if report_metadata["email"] is not None: report_metadata["org_name"] = report_metadata["email"].split("@")[-1] org_name = report_metadata["org_name"] if org_name is not None and " " not in org_name: new_org_name = get_base_domain(org_name) if new_org_name is not None: org_name = new_org_name if not org_name: logger.debug(f"Could not parse org_name from XML.\r\n{report.__str__()}") raise KeyError( "Organization name is missing. This field is a requirement " "for saving the report" ) new_report_metadata["org_name"] = org_name new_report_metadata["org_email"] = report_metadata["email"] # extra_contact_info is langAttrString in RFC 9990 (xs:string in # RFC 7489); its lang attribute is dropped when the report is parsed. extra = _text(report_metadata.get("extra_contact_info")) new_report_metadata["org_extra_contact_info"] = extra new_report_metadata["report_id"] = report_metadata["report_id"] report_id = new_report_metadata["report_id"] report_id = report_id.replace("<", "").replace(">", "").split("@")[0] new_report_metadata["report_id"] = report_id date_range = report["report_metadata"]["date_range"] begin_ts = int(date_range["begin"].split(".")[0]) end_ts = int(date_range["end"].split(".")[0]) span_seconds = end_ts - begin_ts _check_report_span(span_seconds) # The end is checked as well as the begin: a normalized report's # per-day records each name an index from their own interval start, # and every interval start lies at or before the report's end. try: for stamp, what in ( (begin_ts, "Report begin date"), (end_ts, "Report end date"), ): _check_not_future(datetime.fromtimestamp(stamp, timezone.utc), what) except ValueError as error: raise InvalidAggregateReport(str(error)) from error normalize_timespan = ( span_seconds > cfg.normalize_timespan_threshold_hours * 3600 ) date_range["begin"] = timestamp_to_human(begin_ts) date_range["end"] = timestamp_to_human(end_ts) new_report_metadata["begin_date"] = date_range["begin"] new_report_metadata["end_date"] = date_range["end"] new_report_metadata["timespan_requires_normalization"] = normalize_timespan new_report_metadata["original_timespan_seconds"] = span_seconds begin_dt = human_timestamp_to_datetime( new_report_metadata["begin_date"], to_utc=True ) end_dt = human_timestamp_to_datetime( new_report_metadata["end_date"], to_utc=True ) # <error> is langAttrString in RFC 9990 (xs:string in RFC 7489) and # was cardinality-narrowed from "unbounded" to "1" in RFC 9990, but # the parser still accepts a list for backward compatibility with # RFC 7489 reports that carry multiple errors. if "error" in report["report_metadata"]: raw_errors = report["report_metadata"]["error"] if not isinstance(raw_errors, list): raw_errors = [raw_errors] errors = [text for text in (_text(e) for e in raw_errors) if text] new_report_metadata["errors"] = errors # <generator> is a plain xs:string in RFC 9990; attributes and child # elements are already removed, and _text() keeps it a string. generator = _text(report_metadata.get("generator")) new_report_metadata["generator"] = generator new_report["report_metadata"] = new_report_metadata policy_published = report["policy_published"] if type(policy_published) is list: policy_published = policy_published[0] # Final RFC 9990 detection: the dmarc-2.0 XML namespace OR any # RFC 9990-only field. Real-world reporters that follow the schema # without declaring the namespace still get RFC 9990-aware # warnings (missing DKIM selector, removed override-reason types, # etc.) and a truthful audit trail in `xml_namespace`. rfc_9990_only_policy_fields = {"np", "testing", "discovery_method"} is_rfc_9990 = ( xml_namespace == RFC_9990_NAMESPACE or "generator" in report_metadata or any(f in policy_published for f in rfc_9990_only_policy_fields) ) new_policy_published: dict[str, Any] = {} new_policy_published["domain"] = policy_published["domain"] adkim = "r" if "adkim" in policy_published: if policy_published["adkim"] is not None: adkim = policy_published["adkim"] new_policy_published["adkim"] = adkim aspf = "r" if "aspf" in policy_published: if policy_published["aspf"] is not None: aspf = policy_published["aspf"] new_policy_published["aspf"] = aspf new_policy_published["p"] = policy_published["p"] sp = new_policy_published["p"] if "sp" in policy_published: if policy_published["sp"] is not None: sp = policy_published["sp"] new_policy_published["sp"] = sp pct = None if "pct" in policy_published: if policy_published["pct"] is not None: pct = policy_published["pct"] new_policy_published["pct"] = pct fo = None if "fo" in policy_published: if policy_published["fo"] is not None: fo = policy_published["fo"] new_policy_published["fo"] = fo np_ = None if "np" in policy_published: if policy_published["np"] is not None: np_ = policy_published["np"] if np_ not in ("none", "quarantine", "reject"): logger.warning(f"Invalid np value: {sanitize_for_log(str(np_))}") new_policy_published["np"] = np_ testing = None if "testing" in policy_published: if policy_published["testing"] is not None: testing = policy_published["testing"] if testing not in ("n", "y"): logger.warning( f"Invalid testing value: {sanitize_for_log(str(testing))}" ) new_policy_published["testing"] = testing discovery_method = None if "discovery_method" in policy_published: if policy_published["discovery_method"] is not None: discovery_method = policy_published["discovery_method"] if discovery_method not in ("psl", "treewalk"): logger.warning( f"Invalid discovery_method value: " f"{sanitize_for_log(str(discovery_method))}" ) new_policy_published["discovery_method"] = discovery_method new_report["policy_published"] = new_policy_published normalize_timespan = _normalize_within_limit( normalize_timespan, report["record"], begin_dt, end_dt, report_id ) new_report_metadata["timespan_requires_normalization"] = normalize_timespan records = _parse_report_records( report["record"], config=cfg, is_rfc_9990=is_rfc_9990, begin_dt=begin_dt, end_dt=end_dt, normalize=normalize_timespan, keep_alive=keep_alive, dns_budget=_ReverseDNSBudget(), ) new_report["records"] = records return cast(AggregateReport, new_report) except InvalidAggregateReport: # Already specific (an XML or date range limit); do not re-wrap it # as an "Unexpected error" below. raise except expat.ExpatError as error: raise InvalidAggregateReport(f"Invalid XML: {error.__str__()}") from error except KeyError as error: raise InvalidAggregateReport(f"Missing field: {error.__str__()}") from error except AttributeError as error: raise InvalidAggregateReport("Report missing required section") from error except Exception as error: raise InvalidAggregateReport( f"Unexpected error: {error.__str__()}{_exc_origin(error)}" ) from error
def _decompress_gzip_bounded(data: bytes) -> bytes: """ Decompresses a gzip stream, refusing to produce more than ``MAX_DECOMPRESSED_REPORT_SIZE`` bytes. The limit is read from the module global on every call so that it stays a single source of truth (and can be patched in tests). Args: data: The gzip stream Returns: bytes: The decompressed data Raises: ParserError: The decompressed data exceeds the limit, or the gzip stream ends before the decompressor reaches its end marker. zlib.error: The stream is corrupt (bad header, deflate data, or CRC); ``extract_report()`` wraps it in ``ParserError``. """ limit = MAX_DECOMPRESSED_REPORT_SIZE decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16) decompressed = decompressor.decompress(data, max_length=limit + 1) if len(decompressed) > limit: raise ParserError(f"Decompressed report exceeds the {limit} byte limit") # ``flush()`` ignores ``max_length``, so it must not run until the # bounded read above has been accepted. It is bounded here: a bounded # ``decompress()`` that returns fewer than ``max_length`` bytes has # either consumed all of its input or hit the end of the stream (after # which any trailing bytes are ``unused_data``, not input), so nothing # unbounded is left for ``flush()`` to produce. decompressed += decompressor.flush() if not decompressor.eof: # A one-shot zlib.decompress() raises on a stream that ends early; # a decompressobj just stops, so the truncation is detected here. raise ParserError("Incomplete or truncated gzip stream") return decompressed
[docs] def extract_report(content: bytes | str | BinaryIO) -> str: """ Extracts report text from zip- or gzip-compressed content, and returns plain XML or JSON content decoded as-is. A caller-supplied file-like object is read from but never closed by this function, on either the success or the exception path; it is left open and positioned wherever the function's own reads left it, so the caller is free to seek(0) and reuse it. A seekable stream is seeked to position 0 unconditionally (not back to wherever the caller had left it) after its 6-byte header is sniffed, and then read from there, so a successful call typically leaves it at EOF. A non-seekable stream is drained into a buffer this function creates and closes itself; the caller's stream is left open but exhausted. Args: content: The report as a base64-encoded string, file-like object, or bytes. A string that is not valid base64 is returned unchanged. Returns: str: The extracted text """ file_object: BinaryIO | None = None # True while file_object is a BytesIO this function created itself # (from a str/bytes input, or as a buffer copied from a non-seekable # caller stream); it is closed in the ``finally`` below. It is set to # False when file_object instead aliases a caller-supplied seekable # stream, which this function must never close. owns_file_object = True header: bytes try: if isinstance(content, str): try: file_object = BytesIO( b64decode( content.replace("\n", "").replace("\r", ""), validate=True ) ) except binascii.Error: return content header = file_object.read(6) file_object.seek(0) elif isinstance(content, (bytes)): file_object = BytesIO(bytes(content)) header = file_object.read(6) file_object.seek(0) else: stream = cast(BinaryIO, content) seekable = getattr(stream, "seekable", None) can_seek = False if callable(seekable): try: can_seek = bool(seekable()) except Exception: can_seek = False if can_seek: header_raw = stream.read(6) if isinstance(header_raw, str): raise ParserError("File objects must be opened in binary (rb) mode") header = bytes(header_raw) stream.seek(0) # file_object aliases the caller's own stream here, so it # must not be closed by this function. file_object = stream owns_file_object = False else: header_raw = stream.read(6) if isinstance(header_raw, str): raise ParserError("File objects must be opened in binary (rb) mode") header = bytes(header_raw) remainder = stream.read() file_object = BytesIO(header + bytes(remainder)) if header[: len(MAGIC_ZIP)] == MAGIC_ZIP: _zip = zipfile.ZipFile(file_object) limit = MAX_DECOMPRESSED_REPORT_SIZE member = _zip.open(_zip.namelist()[0]).read(limit + 1) if len(member) > limit: raise ParserError(f"Decompressed report exceeds the {limit} byte limit") report = member.decode(errors="ignore") elif header[: len(MAGIC_GZIP)] == MAGIC_GZIP: report = _decompress_gzip_bounded(file_object.read()).decode( errors="ignore" ) elif ( header[: len(MAGIC_XML)] == MAGIC_XML or header[: len(MAGIC_XML_TAG)] == MAGIC_XML_TAG or header[: len(MAGIC_JSON)] == MAGIC_JSON ): # An uncompressed report is held to the decompressed cap, read # at most one byte past it rather than decoded whole. limit = MAX_DECOMPRESSED_REPORT_SIZE data = file_object.read(limit + 1) if len(data) > limit: raise ParserError(f"Report exceeds the {limit} byte limit") report = data.decode(errors="ignore") else: raise ParserError("Not a valid zip, gzip, json, or xml file") except Exception as error: raise ParserError( f"Invalid archive file: {error.__str__()}{_exc_origin(error)}" ) from error finally: if file_object and owns_file_object: try: file_object.close() except Exception: # Best-effort close; a failure here shouldn't shadow # whatever exception (or successful result) is already # propagating out of this function. pass return report
[docs] def extract_report_from_file_path( file_path: str | bytes | os.PathLike[str] | os.PathLike[bytes], ) -> str: """Extracts report from a file at the given file_path""" try: with open(os.fspath(file_path), "rb") as report_file: return extract_report(report_file.read()) except FileNotFoundError: raise ParserError("File was not found")
[docs] def parse_aggregate_report_file( _input: str | bytes | BinaryIO, *, offline: bool = False, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, ip_db_path: str | None = None, nameservers: list[str] | None = None, dns_timeout: float = DEFAULT_DNS_TIMEOUT, dns_retries: int = DEFAULT_DNS_MAX_RETRIES, keep_alive: Callable | None = None, normalize_timespan_threshold_hours: float = 24.0, config: ParserConfig | None = None, ) -> AggregateReport: """Parses a file at the given path, a file-like object, or bytes as an aggregate DMARC report ``_input`` is forwarded to ``extract_report()`` unchanged, so a caller-supplied file-like object is read from but never closed, on either the success or the exception path; see ``extract_report()`` for exactly how such an object is left positioned afterward. Args: _input (str | bytes | IO): A path to a file, a file-like object, or bytes offline (bool): Do not query online for geolocation or DNS always_use_local_files (bool): Do not download files reverse_dns_map_path (str): Path to a reverse DNS map file reverse_dns_map_url (str): URL to a reverse DNS map file ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP nameservers (list): A list of one or more nameservers to use (Cloudflare's public DNS resolvers by default) dns_timeout (float): Sets the DNS timeout in seconds dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors keep_alive (callable): Keep alive function. Not part of ``config``; always applies. normalize_timespan_threshold_hours (float): Normalize timespans beyond this config (ParserConfig): a single object carrying all parsing and enrichment options plus the caches; when provided, the individual option keyword arguments listed above are ignored in favor of the config's values. Returns: dict: The parsed DMARC aggregate report """ cfg = _resolve_config( config, offline=offline, ip_db_path=ip_db_path, always_use_local_files=always_use_local_files, reverse_dns_map_path=reverse_dns_map_path, reverse_dns_map_url=reverse_dns_map_url, nameservers=nameservers, dns_timeout=dns_timeout, dns_retries=dns_retries, normalize_timespan_threshold_hours=normalize_timespan_threshold_hours, ) try: xml = extract_report(_input) except Exception as e: raise InvalidAggregateReport(str(e) + _exc_origin(e)) from e return parse_aggregate_report_xml( xml, config=cfg, keep_alive=keep_alive, )
[docs] def parsed_aggregate_reports_to_csv_rows( reports: AggregateReport | list[AggregateReport], ) -> list[dict[str, Any]]: """ Converts one or more parsed aggregate reports to list of dicts in flat CSV format Args: reports: A parsed aggregate report or list of parsed aggregate reports Returns: list: Parsed aggregate report data as a list of dicts in flat CSV format """ def to_str(obj): return str(obj).lower() if isinstance(reports, dict): reports = [reports] rows = [] for report in reports: xml_schema = report["xml_schema"] org_name = report["report_metadata"]["org_name"] org_email = report["report_metadata"]["org_email"] org_extra_contact = report["report_metadata"]["org_extra_contact_info"] report_id = report["report_metadata"]["report_id"] begin_date = report["report_metadata"]["begin_date"] end_date = report["report_metadata"]["end_date"] normalized_timespan = report["report_metadata"][ "timespan_requires_normalization" ] errors = "|".join(report["report_metadata"]["errors"]) domain = report["policy_published"]["domain"] adkim = report["policy_published"]["adkim"] aspf = report["policy_published"]["aspf"] p = report["policy_published"]["p"] sp = report["policy_published"]["sp"] pct = report["policy_published"]["pct"] fo = report["policy_published"]["fo"] np_ = report["policy_published"].get("np", None) testing = report["policy_published"].get("testing", None) discovery_method = report["policy_published"].get("discovery_method", None) report_dict: dict[str, Any] = dict( xml_schema=xml_schema, org_name=org_name, org_email=org_email, org_extra_contact_info=org_extra_contact, report_id=report_id, begin_date=begin_date, end_date=end_date, normalized_timespan=normalized_timespan, errors=errors, domain=domain, adkim=adkim, aspf=aspf, p=p, sp=sp, np=np_, pct=pct, fo=fo, testing=testing, discovery_method=discovery_method, ) for record in report["records"]: row: dict[str, Any] = report_dict.copy() row["begin_date"] = record["interval_begin"] row["end_date"] = record["interval_end"] row["source_ip_address"] = record["source"]["ip_address"] row["source_country"] = record["source"]["country"] row["source_reverse_dns"] = record["source"]["reverse_dns"] row["source_base_domain"] = record["source"]["base_domain"] row["source_name"] = record["source"]["name"] row["source_type"] = record["source"]["type"] row["source_asn"] = record["source"]["asn"] row["source_as_name"] = record["source"]["as_name"] row["source_as_domain"] = record["source"]["as_domain"] row["count"] = record["count"] row["spf_aligned"] = record["alignment"]["spf"] row["dkim_aligned"] = record["alignment"]["dkim"] row["dmarc_aligned"] = record["alignment"]["dmarc"] row["disposition"] = record["policy_evaluated"]["disposition"] policy_override_reasons = list( map( lambda r_: r_.get("type") or "none", record["policy_evaluated"]["policy_override_reasons"], ) ) policy_override_comments = list( map( lambda r_: r_.get("comment") or "none", record["policy_evaluated"]["policy_override_reasons"], ) ) row["policy_override_reasons"] = ",".join(policy_override_reasons) row["policy_override_comments"] = "|".join(policy_override_comments) row["envelope_from"] = record["identifiers"]["envelope_from"] row["header_from"] = record["identifiers"]["header_from"] envelope_to = record["identifiers"]["envelope_to"] row["envelope_to"] = envelope_to dkim_domains = [] dkim_selectors = [] dkim_results = [] for dkim_result in record["auth_results"]["dkim"]: dkim_domains.append(dkim_result["domain"]) if "selector" in dkim_result: dkim_selectors.append(dkim_result["selector"]) dkim_results.append(dkim_result["result"]) row["dkim_domains"] = ",".join(map(to_str, dkim_domains)) row["dkim_selectors"] = ",".join(map(to_str, dkim_selectors)) row["dkim_results"] = ",".join(map(to_str, dkim_results)) spf_domains = [] spf_scopes = [] spf_results = [] for spf_result in record["auth_results"]["spf"]: spf_domains.append(spf_result["domain"]) spf_scopes.append(spf_result["scope"]) spf_results.append(spf_result["result"]) row["spf_domains"] = ",".join(map(to_str, spf_domains)) row["spf_scopes"] = ",".join(map(to_str, spf_scopes)) row["spf_results"] = ",".join(map(to_str, spf_results)) rows.append(row) for r in rows: for k, v in r.items(): if type(v) not in [str, int, bool]: r[k] = "" return rows
[docs] def parsed_aggregate_reports_to_csv( reports: AggregateReport | list[AggregateReport], ) -> str: """ Converts one or more parsed aggregate reports to flat CSV format, including headers Args: reports: A parsed aggregate report or list of parsed aggregate reports Returns: str: Parsed aggregate report data in flat CSV format, including headers """ fields = [ "xml_schema", "org_name", "org_email", "org_extra_contact_info", "report_id", "begin_date", "end_date", "normalized_timespan", "errors", "domain", "adkim", "aspf", "p", "sp", "np", "pct", "fo", "testing", "discovery_method", "source_ip_address", "source_country", "source_reverse_dns", "source_base_domain", "source_name", "source_type", "source_asn", "source_as_name", "source_as_domain", "count", "spf_aligned", "dkim_aligned", "dmarc_aligned", "disposition", "policy_override_reasons", "policy_override_comments", "envelope_from", "header_from", "envelope_to", "dkim_domains", "dkim_selectors", "dkim_results", "spf_domains", "spf_scopes", "spf_results", ] csv_file_object = StringIO(newline="\n") writer = DictWriter(csv_file_object, fields) writer.writeheader() rows = parsed_aggregate_reports_to_csv_rows(reports) for row in rows: writer.writerow({key: _csv_safe(value) for key, value in row.items()}) csv_file_object.flush() return csv_file_object.getvalue()
[docs] def parse_failure_report( feedback_report: str, sample: str, msg_date: datetime, *, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, offline: bool = False, ip_db_path: str | None = None, nameservers: list[str] | None = None, dns_timeout: float = DEFAULT_DNS_TIMEOUT, dns_retries: int = DEFAULT_DNS_MAX_RETRIES, strip_attachment_payloads: bool = False, config: ParserConfig | None = None, ) -> FailureReport: """ Converts a DMARC failure report and sample to a dict Args: feedback_report (str): A message's feedback report as a string sample (str): The RFC 822 headers or RFC 822 message sample msg_date (datetime): The message's date, used as the arrival date when the feedback report has no ``Arrival-Date`` field ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP always_use_local_files (bool): Do not download files reverse_dns_map_path (str): Path to a reverse DNS map file reverse_dns_map_url (str): URL to a reverse DNS map file offline (bool): Do not query online for geolocation or DNS nameservers (list): A list of one or more nameservers to use (Cloudflare's public DNS resolvers by default) dns_timeout (float): Sets the DNS timeout in seconds dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from failure report results config (ParserConfig): a single object carrying all parsing and enrichment options plus the caches; when provided, the individual option keyword arguments listed above are ignored in favor of the config's values. Returns: dict: A parsed report and sample """ cfg = _resolve_config( config, offline=offline, ip_db_path=ip_db_path, always_use_local_files=always_use_local_files, reverse_dns_map_path=reverse_dns_map_path, reverse_dns_map_url=reverse_dns_map_url, nameservers=nameservers, dns_timeout=dns_timeout, dns_retries=dns_retries, strip_attachment_payloads=strip_attachment_payloads, ) delivery_results = ["delivered", "spam", "policy", "reject", "other"] try: parsed_report: dict[str, Any] = {} report_values = feedback_report_regex.findall(feedback_report) for report_value in report_values: key = report_value[0].lower().replace("-", "_") parsed_report[key] = report_value[1] if "arrival_date" not in parsed_report: if msg_date is None: raise InvalidFailureReport("Failure sample is not a valid email") parsed_report["arrival_date"] = msg_date.isoformat() if "version" not in parsed_report: parsed_report["version"] = None if "user_agent" not in parsed_report: parsed_report["user_agent"] = None if "delivery_result" not in parsed_report: parsed_report["delivery_result"] = None else: for delivery_result in delivery_results: if delivery_result in parsed_report["delivery_result"].lower(): parsed_report["delivery_result"] = delivery_result break if parsed_report["delivery_result"] not in delivery_results: parsed_report["delivery_result"] = "other" try: arrival_utc = human_timestamp_to_datetime( parsed_report["arrival_date"], to_utc=True ) except (ValueError, OverflowError) as error: raise InvalidFailureReport( f"Invalid Arrival-Date: {sanitize_for_log(str(error))}" ) from error try: _check_not_future(arrival_utc, "Arrival-Date") except ValueError as error: raise InvalidFailureReport(str(error)) from error arrival_utc = arrival_utc.strftime("%Y-%m-%d %H:%M:%S") parsed_report["arrival_date_utc"] = arrival_utc ip_address = _strip_zone_id( re.split(r"\s", parsed_report["source_ip"]).pop(0), "Source-IP" ) parsed_report_source = get_ip_address_info( ip_address, cache=cfg.ip_address_cache, ip_db_path=cfg.ip_db_path, always_use_local_files=cfg.always_use_local_files, reverse_dns_map_path=cfg.reverse_dns_map_path, reverse_dns_map_url=cfg.reverse_dns_map_url, reverse_dns_map=cfg.reverse_dns_map, offline=cfg.offline, nameservers=cfg.nameservers, timeout=cfg.dns_timeout, retries=cfg.dns_retries, psl_overrides_path=cfg.psl_overrides_path, psl_overrides_url=cfg.psl_overrides_url, ) parsed_report["source"] = parsed_report_source del parsed_report["source_ip"] # Identity-Alignment is REQUIRED per RFC 9991 §4. Default silently for # backward compatibility with pre-9991 reporters, but log so the # offending reporter is visible. Values are CFWS-separated per the # ABNF, so each mechanism is stripped after splitting. if "identity_alignment" not in parsed_report: logger.warning( "Failure report missing required 'Identity-Alignment' " "field (RFC 9991 §4); defaulting to no aligned mechanisms" ) parsed_report["authentication_mechanisms"] = [] else: raw_alignment = parsed_report["identity_alignment"].strip() if raw_alignment.lower() == "none": parsed_report["authentication_mechanisms"] = [] else: parsed_report["authentication_mechanisms"] = [ m.strip() for m in raw_alignment.split(",") if m.strip() ] del parsed_report["identity_alignment"] # Auth-Failure is REQUIRED per RFC 9991 §4. Comma-separated per ABNF # so strip each token. if "auth_failure" not in parsed_report: logger.warning( "Failure report missing required 'Auth-Failure' field " "(RFC 9991 §4); defaulting to 'dmarc'" ) parsed_report["auth_failure"] = "dmarc" parsed_report["auth_failure"] = [ f.strip() for f in parsed_report["auth_failure"].split(",") if f.strip() ] # Feedback-Type is REQUIRED per RFC 5965 §3.1, but some gateways # (e.g. Exim/cPanel-based ones that send a plain-text summary without # a machine-readable message/feedback-report part) omit it. The # Elasticsearch/OpenSearch outputs require the key, so default it # instead of dropping the report there (see issue #332). if "feedback_type" not in parsed_report: logger.warning( "Failure report missing required 'Feedback-Type' field " "(RFC 5965 §3.1); defaulting to 'auth-failure'" ) parsed_report["feedback_type"] = "auth-failure" # Authentication-Results is likewise REQUIRED per RFC 6591 §3.1 and # required by the Elasticsearch/OpenSearch outputs. if "authentication_results" not in parsed_report: logger.warning( "Failure report missing required 'Authentication-Results' " "field (RFC 6591 §3.1); defaulting to None" ) parsed_report["authentication_results"] = None optional_fields = [ "original_envelope_id", "dkim_domain", "original_mail_from", "original_rcpt_to", ] for optional_field in optional_fields: if optional_field not in parsed_report: parsed_report[optional_field] = None parsed_sample = parse_email( sample, strip_attachment_payloads=cfg.strip_attachment_payloads ) if "reported_domain" not in parsed_report: # reported_domain is a required str in the FailureReport contract # (parsedmarc/types.py) and is read directly by the Elasticsearch # and OpenSearch outputs, so it cannot be defaulted to None. When # the report omits the Reported-Domain field and the sample's From # header is missing or unparseable, the report is unusable. sample_from = parsed_sample.get("from") if not isinstance(sample_from, dict) or not sample_from.get("domain"): raise InvalidFailureReport( "The failure report has no Reported-Domain field, and no " "domain could be parsed from the From header of the " "included sample message" ) parsed_report["reported_domain"] = sample_from["domain"] sample_headers_only = False number_of_attachments = len(parsed_sample["attachments"]) if number_of_attachments < 1 and parsed_sample["body"] is None: sample_headers_only = True if sample_headers_only and parsed_sample["has_defects"]: del parsed_sample["defects"] del parsed_sample["defects_categories"] del parsed_sample["has_defects"] parsed_report["sample_headers_only"] = sample_headers_only parsed_report["sample"] = sample parsed_report["parsed_sample"] = parsed_sample return cast(FailureReport, parsed_report) except InvalidFailureReport: # Already a clear, specific message; do not re-wrap it as an # "Unexpected error" below. raise except KeyError as error: raise InvalidFailureReport(f"Missing value: {error.__str__()}") from error except MessageLimitExceeded as error: raise InvalidFailureReport(f"Sample message rejected: {error}") from error except Exception as error: raise InvalidFailureReport( f"Unexpected error: {error.__str__()}{_exc_origin(error)}" ) from error
[docs] def parsed_failure_reports_to_csv_rows( reports: FailureReport | list[FailureReport], ) -> list[dict[str, Any]]: """ Converts one or more parsed failure reports to a list of dicts in flat CSV format Args: reports: A parsed failure report or list of parsed failure reports Returns: list: Parsed failure report data as a list of dicts in flat CSV format """ if isinstance(reports, dict): reports = [reports] rows = [] for report in reports: row: dict[str, Any] = dict(report) row["source_ip_address"] = report["source"]["ip_address"] row["source_reverse_dns"] = report["source"]["reverse_dns"] row["source_base_domain"] = report["source"]["base_domain"] row["source_name"] = report["source"]["name"] row["source_type"] = report["source"]["type"] row["source_asn"] = report["source"]["asn"] row["source_as_name"] = report["source"]["as_name"] row["source_as_domain"] = report["source"]["as_domain"] row["source_country"] = report["source"]["country"] del row["source"] row["subject"] = report["parsed_sample"].get("subject") row["auth_failure"] = ",".join(report["auth_failure"]) authentication_mechanisms = report["authentication_mechanisms"] row["authentication_mechanisms"] = ",".join(authentication_mechanisms) del row["sample"] del row["parsed_sample"] rows.append(row) return rows
[docs] def parsed_failure_reports_to_csv( reports: FailureReport | list[FailureReport], ) -> str: """ Converts one or more parsed failure reports to flat CSV format, including headers Args: reports: A parsed failure report or list of parsed failure reports Returns: str: Parsed failure report data in flat CSV format, including headers """ fields = [ "feedback_type", "user_agent", "version", "original_envelope_id", "original_mail_from", "original_rcpt_to", "arrival_date", "arrival_date_utc", "subject", "message_id", "authentication_results", "dkim_domain", "source_ip_address", "source_country", "source_reverse_dns", "source_base_domain", "source_name", "source_type", "source_asn", "source_as_name", "source_as_domain", "delivery_result", "auth_failure", "reported_domain", "authentication_mechanisms", "sample_headers_only", ] csv_file = StringIO() csv_writer = DictWriter(csv_file, fieldnames=fields) csv_writer.writeheader() rows = parsed_failure_reports_to_csv_rows(reports) for row in rows: new_row: dict[str, Any] = {} for key in fields: new_row[key] = _csv_safe(row.get(key)) csv_writer.writerow(new_row) return csv_file.getvalue()
def _decode_mime_payload(part: email.message.Message, fallback: str) -> str: """ Returns a MIME part's payload decoded per its ``Content-Transfer-Encoding`` Three non-obvious constraints shape this helper: - Only ``quoted-printable`` and ``base64`` parts are touched at all. The caller parses the message from a ``str``, so for a part with a 7bit, 8bit, or absent ``Content-Transfer-Encoding`` there is nothing to undo, and ``compat32``'s ``get_payload(decode=True)`` would round-trip the already-correct text through ``raw-unicode-escape`` bytes — re-decoding those as the declared charset mangles every non-ASCII character. Such parts return ``fallback``, which is the payload text as-is. - The standard library's email parser nests **every** ``message/*`` subtype — including ``message/feedback-report`` and ``message/rfc822`` — as a child ``Message`` object, so ``get_payload(decode=True)`` returns ``None`` for those parts rather than decoded bytes. The caller's re-serialization of that child message is passed in as ``fallback``, and any transfer encoding is undone here instead. - A composite part carrying a real ``Content-Transfer-Encoding`` of ``quoted-printable`` or ``base64`` is illegal per RFC 2045 §6.4, but real reporters send them anyway (issue `#882 <https://github.com/domainaware/parsedmarc/issues/882>`_), and the standard library nests the still-encoded text as-is. Parsing that text as a message can insert a header/body separator that the encoded text never had, which has to be undone before decoding; see below. Any unexpected failure returns ``fallback`` unchanged, so a report that parsed before cannot start failing because of this decoding step. Args: part: The MIME part fallback: The payload text to use when the part carries no transfer encoding to undo, or cannot be decoded with ``decode=True`` (i.e. nested ``message/*`` parts) Returns: str: The decoded payload """ try: cte = (part.get("Content-Transfer-Encoding") or "").strip().lower() if cte not in ("quoted-printable", "base64"): return fallback payload_bytes = part.get_payload(decode=True) if isinstance(payload_bytes, bytes): charset = part.get_content_charset() or "utf-8" try: return payload_bytes.decode(charset, errors="replace") except LookupError: return payload_bytes.decode("utf-8", errors="replace") # Nested message/* part: undo the transfer encoding by hand. # # The still-encoded text was parsed as a message first. A soft line # break (RFC 2045 §6.7) splits a long field across lines, and the # continuation line has neither a colon nor leading whitespace, so the # parser records a MissingHeaderBodySeparatorDefect and treats the # remainder as the body -- which makes the re-serialized `fallback` # carry a blank line that the encoded text never had. Decoding then # consumes the soft break and leaves that inserted newline in the # middle of the value, truncating it. Drop exactly that one separator: # a header value cannot contain a blank line, so the first "\n\n" is # the inserted boundary. Text with a genuine separator raises no # defect, and nothing is removed. text = fallback child = part.get_payload() if isinstance(child, list) and child: child = child[0] if isinstance(child, email.message.Message) and any( isinstance(defect, email.errors.MissingHeaderBodySeparatorDefect) for defect in child.defects ): text = fallback.replace("\n\n", "\n", 1) if cte == "quoted-printable": return quopri.decodestring(text.encode("utf-8", errors="replace")).decode( "utf-8", errors="replace" ) return b64decode(text).decode("utf-8", errors="replace") except Exception: return fallback
[docs] def parse_report_email( input_: bytes | str, *, offline: bool = False, ip_db_path: str | None = None, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, nameservers: list[str] | None = None, dns_timeout: float = DEFAULT_DNS_TIMEOUT, dns_retries: int = DEFAULT_DNS_MAX_RETRIES, strip_attachment_payloads: bool = False, keep_alive: Callable | None = None, normalize_timespan_threshold_hours: float = 24.0, config: ParserConfig | None = None, ) -> ParsedReport: """ Parses a DMARC or SMTP TLS report from an email Args: input_: An emailed DMARC report in RFC 822 format, as bytes or a string ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP always_use_local_files (bool): Do not download files reverse_dns_map_path (str): Path to a reverse DNS map reverse_dns_map_url (str): URL to a reverse DNS map offline (bool): Do not query online for geolocation or DNS nameservers (list): A list of one or more nameservers to use dns_timeout (float): Sets the DNS timeout in seconds dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from failure report results keep_alive (callable): keep alive function. Not part of ``config``; always applies. normalize_timespan_threshold_hours (float): Normalize timespans beyond this config (ParserConfig): a single object carrying all parsing and enrichment options plus the caches; when provided, the individual option keyword arguments listed above are ignored in favor of the config's values. Returns: dict: * ``report_type``: ``aggregate``, ``failure``, or ``smtp_tls`` * ``report``: The parsed report Raises: ParserError: If the message is not a valid report. Whatever the input, only ``ParserError`` subclasses are raised. A message that is not a valid report, or whose attachment is not one, raises ``InvalidDMARCReport``, or ``InvalidSMTPTLSReport`` for an SMTP TLS attachment (both are ``ParserError`` subclasses). Any other failure to parse the message raises a plain ``ParserError``. """ cfg = _resolve_config( config, offline=offline, ip_db_path=ip_db_path, always_use_local_files=always_use_local_files, reverse_dns_map_path=reverse_dns_map_path, reverse_dns_map_url=reverse_dns_map_url, nameservers=nameservers, dns_timeout=dns_timeout, dns_retries=dns_retries, strip_attachment_payloads=strip_attachment_payloads, normalize_timespan_threshold_hours=normalize_timespan_threshold_hours, ) result: ParsedReport | None = None msg_date: datetime = datetime.now(timezone.utc) try: input_data: str | bytes | bytearray | memoryview = input_ if isinstance(input_data, (bytes, bytearray, memoryview)): input_bytes = bytes(input_data) if is_outlook_msg(input_bytes): converted = convert_outlook_msg(input_bytes) if isinstance(converted, str): input_str = converted else: input_str = bytes(converted).decode( encoding="utf8", errors="replace" ) else: input_str = input_bytes.decode(encoding="utf8", errors="replace") else: input_str = input_data # Reject a message whose structure makes parsing slow before any # parser builds its tree (GHSA-qcc2-4x3v-v4g7) check_message_limits(input_str) msg = mailparser.parse_from_string(input_str) msg_headers = json.loads(msg.headers_json) if "Date" in msg_headers: try: msg_date = human_timestamp_to_datetime(msg_headers["Date"]) except (ValueError, OverflowError) as e: raise InvalidDMARCReport( f"Invalid Date header: {sanitize_for_log(str(e))}" ) from e date = email.utils.format_datetime(msg_date) msg = email.message_from_string(input_str) except MessageLimitExceeded as e: raise InvalidDMARCReport(f"Message rejected: {e}") from e except InvalidDMARCReport: raise except Exception as e: raise ParserError(e.__str__() + _exc_origin(e)) from e subject_text = "None" feedback_report = None smtp_tls_report = None sample = None # id() of every sample candidate and every part inside one sample_parts: set[int] = set() is_feedback_report: bool = False if "From" in msg_headers: logger.info( "Parsing mail from {} on {}".format( sanitize_for_log(str(msg_headers["From"])), date ) ) if "Subject" in msg_headers: # RFC 2047 decoding can yield raw CR/LF, so escape control # characters before the subject reaches an error message or log. subject_text = sanitize_for_log(str(msg_headers["Subject"])) # Serializing a part (``str(payload_obj[0])``) re-renders any # ``message/rfc822`` sub-message, which can raise e.g. # ``email.errors.HeaderWriteError`` on a control character in a header or # ``RecursionError`` on deeply nested multiparts. Callers rely on this # function raising only ``ParserError`` subclasses so that one bad message # is skipped (and moved to ``Invalid``) rather than aborting a whole # mailbox batch (GHSA-qxwv-2vwj-v828). try: for part in msg.walk(): if id(part) in sample_parts: # The sample is the reported message, spoofer's content # included; an attachment inside it is not a report. continue content_type = part.get_content_type().lower() payload_obj = part.get_payload() if not isinstance(payload_obj, list): payload_obj = [payload_obj] if content_type.startswith("multipart/") or content_type == "text/html": continue # A part is serialized only where its text is used: str() of a # composite part renders its whole subtree, and walk() visits # every level, so serializing each one was quadratic in the # nesting (98 multiparts around a 20 MiB leaf took 26.6 s; # GHSA-qcc2-4x3v-v4g7). if is_feedback_report and content_type in EMAIL_SAMPLE_CONTENT_TYPES: # A feedback report has one sample (RFC 6591 §3.1); a message # nested inside it belongs to it and must not replace it. if sample is None: sample = _decode_mime_payload(part, str(payload_obj[0])) # A later candidate is not read as the sample, and neither # are the parts inside it. sample_parts.update(id(sample_part) for sample_part in part.walk()) continue if content_type == "message/feedback-report" and is_feedback_report: # A feedback report has one message/feedback-report part # (RFC 6591 §3); a nested one must not be rendered again. continue if content_type != "message/feedback-report" and isinstance( payload_obj[0], email.message.Message ): # Another message/* body; walk() visits its parts. continue payload = str(payload_obj[0]) if content_type == "message/feedback-report": is_feedback_report = True # The RFC 6591 part wins over a legacy text report read # earlier; its sample follows it. sample = None decoded_payload = _decode_mime_payload(part, payload) try: if "Feedback-Type" in decoded_payload: feedback_report = decoded_payload else: feedback_report = b64decode(decoded_payload).__str__() feedback_report = feedback_report.lstrip("b'").rstrip("'") feedback_report = feedback_report.replace("\\r", "") feedback_report = feedback_report.replace("\\n", "\n") except (ValueError, TypeError, binascii.Error): feedback_report = decoded_payload elif content_type == "application/tlsrpt+json": if not payload.strip().startswith("{"): payload = b64decode(payload).decode("utf-8", errors="replace") smtp_tls_report = parse_smtp_tls_report_json(payload) return {"report_type": "smtp_tls", "report": smtp_tls_report} elif content_type == "application/tlsrpt+gzip": payload = extract_report(payload) smtp_tls_report = parse_smtp_tls_report_json(payload) return {"report_type": "smtp_tls", "report": smtp_tls_report} elif content_type == "text/plain": # A legacy text report has no message/feedback-report part. # Once one is read, a text part (which may sit inside the # sample, the spoofer's own text) must not replace the # report's Source-IP or its sample. if ( not is_feedback_report and "A message claiming to be from you has failed" in payload ): try: parts = payload.split("detected.", 1) field_matches = text_report_regex.findall(parts[0]) fields = dict() for match in field_matches: field_name = match[0].lower().replace(" ", "-") fields[field_name] = match[1].strip() feedback_report = "Arrival-Date: {}\nSource-IP: {}".format( fields["received-date"], fields["sender-ip-address"] ) except Exception as e: error = f'Unable to parse message with subject "{subject_text}": {e}{_exc_origin(e)}' raise InvalidDMARCReport(error) from e sample = parts[1].lstrip() logger.debug(sample) else: try: payload_bytes = b64decode(payload) if payload_bytes.startswith(MAGIC_ZIP) or payload_bytes.startswith( MAGIC_GZIP ): payload_text = extract_report(payload_bytes) else: payload_text = payload_bytes.decode("utf-8", errors="replace") if payload_text.strip().startswith("{"): smtp_tls_report = parse_smtp_tls_report_json(payload_text) result = {"report_type": "smtp_tls", "report": smtp_tls_report} return result elif payload_text.strip().startswith("<"): aggregate_report = parse_aggregate_report_xml( payload_text, config=cfg, keep_alive=keep_alive, ) result = { "report_type": "aggregate", "report": aggregate_report, } return result except (TypeError, ValueError, binascii.Error): # b64decode() rejected this MIME part's payload as not # base64-decodable, so it isn't a report attachment; fall # through and keep walking the message for a part (or the # feedback-report/sample pair below) that is. pass except InvalidDMARCReport as e: error = f'Message with subject "{subject_text}" is not a valid DMARC report: {e}' raise InvalidDMARCReport(error) from e except InvalidSMTPTLSReport: # Named with the message's subject by the handler below. raise except Exception as e: error = f'Unable to parse message with subject "{subject_text}": {e}{_exc_origin(e)}' raise ParserError(error) from e except InvalidSMTPTLSReport as e: # From either the tlsrpt content-type branches or a JSON attachment. error = ( f'Message with subject "{subject_text}" is not a valid SMTP TLS report: {e}' ) raise InvalidSMTPTLSReport(error) from e except ParserError: raise except Exception as e: error = f'Unable to parse message with subject "{subject_text}": {e}{_exc_origin(e)}' raise ParserError(error) from e if feedback_report and sample: try: failure_report = parse_failure_report( feedback_report, sample, msg_date, config=cfg, ) except InvalidFailureReport as e: error = ( f'Message with subject "{subject_text}" ' "is not a valid " f"failure DMARC report: {e}" ) raise InvalidFailureReport(error) from e result = {"report_type": "failure", "report": failure_report} return result # Every branch that sets ``result`` returns immediately, so reaching this # point always means no report was found. error = f'Message with subject "{subject_text}" is not a valid report' raise InvalidDMARCReport(error)
# An RFC 5322 header field name (printable ASCII excluding the colon) followed # by a colon at the start of a line, or an mbox "From " separator. _email_header_regex = re.compile(r"^(From |[\x21-\x39\x3b-\x7e]+:)") def _looks_like_email(text: str) -> bool: """Returns True if the first line looks like an email header. Callers pass already-``lstrip()``-ed text, so the first line is the first meaningful line. """ first_line = text.split("\n", 1)[0] return _email_header_regex.match(first_line) is not None def _leading_text(content: str | bytes) -> str: """Return the start of ``content`` (512 characters at most) without leading whitespace, to sniff its format.""" if isinstance(content, (bytes, bytearray, memoryview)): sniff = bytes(content)[:512].decode("utf-8", errors="replace") else: sniff = content[:512] return sniff.lstrip() def _describe_parse_failure( content: str | bytes, aggregate_error: str, smtp_tls_error: str, email_error: str, ) -> str: """Builds a human-readable reason for a parse_report_file failure. parse_report_file tries the aggregate XML, SMTP TLS JSON, and report-email parsers in turn; when all three reject the input, only the parser for the format the content actually resembles produced a meaningful error. The other two are noise -- a malformed aggregate report is also "not JSON" and "not an email". Sniff the leading non-whitespace byte to surface the single relevant reason. """ sniff = _leading_text(content) if sniff.startswith("<"): return f"Invalid aggregate report: {aggregate_error}" if sniff.startswith("{"): return f"Invalid SMTP TLS report: {smtp_tls_error}" if _looks_like_email(sniff): return f"Invalid report email: {email_error}" return ( "Not a recognized report format (not a DMARC aggregate XML report, " "an SMTP TLS JSON report, or a DMARC report email)" ) def _read_up_to(file_object: BinaryIO, limit: int) -> bytes: """Read from ``file_object`` until end of file or ``limit`` bytes, whichever comes first. A raw or unbuffered stream (a socket file, a pipe) may return fewer bytes than asked for without being at its end, so one sized ``read()`` could hand the parser a truncated report.""" # One growing buffer, not a list of chunks: a stream that returns a byte # at a time would otherwise cost one bytes object per byte read. content = bytearray() while len(content) < limit: chunk = file_object.read(limit - len(content)) if not chunk: break content += chunk return bytes(content)
[docs] def parse_report_file( input_: bytes | str | os.PathLike[str] | os.PathLike[bytes] | BinaryIO, *, nameservers: list[str] | None = None, dns_timeout: float = DEFAULT_DNS_TIMEOUT, dns_retries: int = DEFAULT_DNS_MAX_RETRIES, strip_attachment_payloads: bool = False, ip_db_path: str | None = None, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, offline: bool = False, keep_alive: Callable | None = None, normalize_timespan_threshold_hours: float = 24.0, config: ParserConfig | None = None, ) -> ParsedReport: """Parses a DMARC aggregate report, DMARC failure report, or SMTP TLS report from a file at the given path, a file-like object, or bytes A path is opened and closed by this function. A caller-supplied file object is read from but never closed, on either the success or the exception path; it is left open and positioned wherever its own ``read()`` left it, so the caller is free to ``seek(0)`` and reuse it. When the input is over the size cap, that is one byte past the cap: no more is read. Args: input_ (str | os.PathLike | bytes | BinaryIO): A path to a file, a file-like object, or bytes nameservers (list): A list of one or more nameservers to use (Cloudflare's public DNS resolvers by default) dns_timeout (float): Sets the DNS timeout in seconds dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from failure report results ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP always_use_local_files (bool): Do not download files reverse_dns_map_path (str): Path to a reverse DNS map reverse_dns_map_url (str): URL to a reverse DNS map offline (bool): Do not make online queries for geolocation or DNS keep_alive (callable): Keep alive function. Not part of ``config``; always applies. normalize_timespan_threshold_hours (float): Normalize timespans beyond this config (ParserConfig): a single object carrying all parsing and enrichment options plus the caches; when provided, the individual option keyword arguments listed above are ignored in favor of the config's values. Returns: dict: The parsed DMARC report Raises: ParserError: if the input is not a valid report, or is larger than ``MAX_DECOMPRESSED_REPORT_SIZE`` (refused before it is read into memory) """ cfg = _resolve_config( config, offline=offline, ip_db_path=ip_db_path, always_use_local_files=always_use_local_files, reverse_dns_map_path=reverse_dns_map_path, reverse_dns_map_url=reverse_dns_map_url, nameservers=nameservers, dns_timeout=dns_timeout, dns_retries=dns_retries, strip_attachment_payloads=strip_attachment_payloads, normalize_timespan_threshold_hours=normalize_timespan_threshold_hours, ) file_object: BinaryIO # An input over the cap is refused before it is held in memory (#949). A # valid compressed report is smaller than its decompressed form, so the # one cap serves both; the message is the one the content check gives. too_large = ParserError(_size_cap_text(b"")) if isinstance(input_, (str, os.PathLike)): file_path = os.fspath(input_) logger.debug(f"Parsing {file_path}") if os.path.getsize(file_path) > MAX_DECOMPRESSED_REPORT_SIZE: raise too_large # A path is a handle we opened ourselves, so it must be closed on # both the success and exception paths. with open(file_path, "rb") as file_object: content = _read_up_to(file_object, MAX_DECOMPRESSED_REPORT_SIZE + 1) else: if isinstance(input_, (bytes, bytearray, memoryview)): if memoryview(input_).nbytes > MAX_DECOMPRESSED_REPORT_SIZE: raise too_large # The BytesIO wrapper is created here, so this function owns # it and closes it once the bytes have been read out. file_object = BytesIO(bytes(input_)) content = file_object.read() file_object.close() else: # A caller-supplied file-like object is never closed by this # function, on success or failure: the caller may want to # seek(0) and retry, inspect tell(), or otherwise reuse the # handle afterward. file_object = input_ content = _read_up_to(file_object, MAX_DECOMPRESSED_REPORT_SIZE + 1) if len(content) > MAX_DECOMPRESSED_REPORT_SIZE: raise too_large if content.startswith(MAGIC_ZIP) or content.startswith(MAGIC_GZIP): content = extract_report(content) # parse_report_file tries the three report formats in turn. When all three # reject the input, keep each format's error text so the final message # can explain *why* the file is invalid instead of a bare "Not a valid # report" (see _describe_parse_failure). Only the text is kept: an # exception's traceback holds the failed parser's frames, and with them # its partial tree, while the next parser runs (GHSA-q7gm-8cj9-94hr). aggregate_message = smtp_tls_message = email_message = "" try: report = parse_aggregate_report_file( content, config=cfg, keep_alive=keep_alive, ) return {"report_type": "aggregate", "report": report} except InvalidAggregateReport as aggregate_error: aggregate_message = str(aggregate_error) leading = _leading_text(content) if leading.startswith("<"): # JSON never begins with "<", and parse_smtp_tls_report_json() would # first decode the whole content. smtp_tls_message = "not JSON" else: try: report = parse_smtp_tls_report_json(content) return {"report_type": "smtp_tls", "report": report} except InvalidSMTPTLSReport as smtp_tls_error: smtp_tls_message = str(smtp_tls_error) if leading.startswith(("<", "{")): # An RFC 822 message begins with a header line, never "<" or "{"; # such a file's failure is reported as the aggregate or SMTP TLS # error, so no email reason is needed. pass elif _exceeds_size_cap(content): # The email parser has no size cap of its own. email_message = _size_cap_text(content) else: try: return parse_report_email(content, config=cfg, keep_alive=keep_alive) except InvalidDMARCReport as email_error: email_message = str(email_error) message = _describe_parse_failure( content, aggregate_message, smtp_tls_message, email_message ) # The cause is a fresh exception with the relevant format's text, so the # chain is kept without the failed parse's traceback. if leading.startswith("<"): raise ParserError(message) from InvalidAggregateReport(aggregate_message) if leading.startswith("{"): raise ParserError(message) from InvalidSMTPTLSReport(smtp_tls_message) raise ParserError(message) from InvalidDMARCReport(email_message)
_ASCII_UPPER_TO_LOWER = str.maketrans( "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" ) def _aggregate_report_dedup_key(report: AggregateReport) -> str: """Return the ``seen_aggregate_report_ids`` cache key for ``report``. RFC 9990 §3.5.1 defines a Report-ID as unique only "among reports to the same domain", so the key is the reporter, the Report-ID, and the policy domain. ASCII domain letters are compared without case per RFC 4343 §3, without changing the domain in the report itself. A JSON array encodes the three fields without the ambiguity of a delimiter that a field could contain, and the key is the SHA-256 hex digest of that JSON, so it is 64 characters long whatever the sender puts in the three fields (GHSA-9fw6-mwqr-p43m: the JSON itself was the key, so a report with a multi-megabyte Report-ID cost that much memory in the cache). Every writer of the shared cache must build its key here -- the mbox and mailbox classifier and the CLI's direct-file path -- or a report read through one path is not recognized as a duplicate of the same report read through another. """ report_org = report["report_metadata"]["org_name"] report_id = report["report_metadata"]["report_id"] report_domain = report["policy_published"]["domain"] # The existing XML parser returns None for an empty <domain/>. if isinstance(report_domain, str): report_domain = report_domain.translate(_ASCII_UPPER_TO_LOWER) encoded = json.dumps((report_org, report_id, report_domain)) return hashlib.sha256(encoded.encode("utf-8")).hexdigest() def _classify_parsed_email( parsed_email: ParsedReport, aggregate_reports: list[AggregateReport], failure_reports: list[FailureReport], smtp_tls_reports: list[SMTPTLSReport], *, seen_aggregate_report_ids: ExpiringDict, pending_aggregate_keys: set[str] | None = None, ) -> ReportType: """Classify a parsed report email, appending it to the matching list. Owns the seen-aggregate-report-ID dedup check against ``seen_aggregate_report_ids``: an aggregate report already seen (keyed on the reporter, the Report-ID, and the reported policy domain) is logged and dropped instead of appended. Shared, unmodified, by the sequential and parallel branches of ``get_dmarc_reports_from_mbox`` and ``get_dmarc_reports_from_mailbox`` so both dedup identically -- callers pass the config's ``seen_aggregate_report_ids`` cache so dedup state stays scoped to whichever ``ParserConfig`` (explicit or module-default) is in effect. The key comes from :func:`_aggregate_report_dedup_key`, which the CLI's direct-file path also uses, so a report seen in a file and again in an mbox in the same run is recognized as a duplicate. ``pending_aggregate_keys`` lets a caller *defer* the dedup-cache write: when a set is supplied, a newly seen key is staged in that set instead of being written to ``seen_aggregate_report_ids``, and the dedup check consults both. The caller then folds the staged keys into the cache once the batch is known to have been saved (see ``get_dmarc_reports_from_mailbox``), or drops them so a retry reparses the same reports rather than silently skipping them as duplicates. ``None`` (the default) keeps the immediate-write behavior. Returns the report type so mailbox callers know which UID list to append the source message's UID to. """ report_type = parsed_email["report_type"] # Compare against parsed_email["report_type"] directly (not the # report_type local above) in each branch so pyright's TypedDict # discriminated-union narrowing applies to parsed_email["report"]. if parsed_email["report_type"] == "aggregate": report_org = parsed_email["report"]["report_metadata"]["org_name"] report_id = parsed_email["report"]["report_metadata"]["report_id"] report_key = _aggregate_report_dedup_key(parsed_email["report"]) already_seen = report_key in seen_aggregate_report_ids or ( pending_aggregate_keys is not None and report_key in pending_aggregate_keys ) if not already_seen: if pending_aggregate_keys is None: seen_aggregate_report_ids[report_key] = True else: pending_aggregate_keys.add(report_key) aggregate_reports.append(parsed_email["report"]) else: logger.debug( "Skipping duplicate aggregate report from " f"{sanitize_for_log(str(report_org))} " f"with ID: {sanitize_for_log(str(report_id))}" ) elif parsed_email["report_type"] == "failure": failure_reports.append(parsed_email["report"]) elif parsed_email["report_type"] == "smtp_tls": smtp_tls_reports.append(parsed_email["report"]) return report_type def _fetch_mailbox_message( connection: MailboxConnection, msg_uid: Any, test: bool ) -> tuple[int | str, str | ParserError]: """Fetch one message from ``connection`` by UID, casting the UID to the type each backend's ``fetch_message`` expects. Shared, unmodified, by the sequential and parallel branches of ``get_dmarc_reports_from_mailbox`` so both fetch identically. Returns ``(message_id, msg_content)``; ``message_id`` is the backend-appropriate id to use for later move/delete calls. A backend that re-serializes the stored message while fetching it (the Maildir backend of mailsuite before 2.3.3 called ``Message.as_string()``; 2.3.3 returns the stored bytes) can fail on the message's own content: ``email.errors.HeaderWriteError`` on a control character in a header, ``RecursionError`` on deeply nested multiparts. Those content errors are returned, not raised, as ``msg_content`` -- a ``ParserError`` the caller treats like a parse failure, disposing of the message as invalid instead of aborting the batch on every run (GHSA-qxwv-2vwj-v828). The catch is deliberately narrow (``email.errors.MessageError`` and ``RecursionError``, not ``Exception``): transport and service errors (``OSError``, IMAP and Microsoft Graph errors, ``httpx`` errors) say nothing about the message and still propagate, because classifying a transient network failure as invalid content would move a legitimate report to the invalid folder. """ message_id: int | str if isinstance(connection, IMAPConnection): message_id = int(msg_uid) else: message_id = str(msg_uid) if not isinstance(msg_uid, str) else msg_uid try: if isinstance(connection, (MSGraphConnection, MaildirConnection)): msg_content = connection.fetch_message(message_id, mark_read=not test) else: msg_content = connection.fetch_message(message_id) except (email.errors.MessageError, RecursionError) as e: return message_id, ParserError( f"Unable to fetch message UID {message_id}: {type(e).__name__}: {e}" ) return message_id, msg_content def _dispose_invalid_message( connection: MailboxConnection, message_id: int | str, delete: bool, invalid_reports_folder: str, ) -> bool: """Delete or move an unparseable message, per ``delete``. Callers pass their effective ``delete_invalid`` value as ``delete``. Returns whether the message left the reports folder. Shared, unmodified, by the sequential and parallel branches of ``get_dmarc_reports_from_mailbox`` so both dispose of invalid messages identically. A delete or move the backend rejects is logged and the message is left in place, as the archival disposal of parsed reports does, rather than aborting the rest of the batch. Besides transient backend errors, this covers messages the backend cannot rewrite: the Maildir backend of mailsuite before 2.3.3 moved a message by storing it again with ``mailbox.Maildir.add()``, which re-serializes it and raises ``RecursionError`` for deeply nested multiparts (and ``email.errors.HeaderWriteError`` for a control character in a header where ``BytesGenerator`` verifies headers), so such a message stayed in the inbox. mailsuite 2.3.3 renames the file instead (GHSA-qxwv-2vwj-v828, GHSA-9jx3-p8wm-g8vw). """ try: if delete: logger.debug(f"Deleting message UID {message_id}") if isinstance(connection, IMAPConnection): connection.delete_message(int(message_id)) else: connection.delete_message(str(message_id)) else: logger.debug(f"Moving message UID {message_id} to {invalid_reports_folder}") if isinstance(connection, IMAPConnection): connection.move_message(int(message_id), invalid_reports_folder) else: connection.move_message(str(message_id), invalid_reports_folder) except Exception as e: action = "deleting" if delete else "moving" logger.error( f"Mailbox error: Error {action} invalid message UID {message_id}: " f"{type(e).__name__}: {e}" ) return False return True def _record_parse_attempt( attempt_store: AttemptStore, key: str, message_id: int | str, reports_folder: str, delete_invalid: bool, invalid_reports_folder: str, test: bool, ) -> bool: """Record that a message is about to be parsed; return ``True`` when it must be set aside instead. A message whose earlier parse attempts never finished ``MAX_PARSE_ATTEMPTS`` times is taken to be what killed those runs: the error is logged here, and the caller disposes of the message as invalid without parsing it (or, under ``test``, leaves it in place). Such a message is not recorded again, so a message left in place by a failed move or by ``test`` keeps the same count on every later run. """ count = attempt_store.count(key) if not attempt_store.enabled or count < MAX_PARSE_ATTEMPTS: attempt_store.record(key) return False if test: action = "Leaving it in place without parsing it (test mode)" elif delete_invalid: action = "Deleting it without parsing it" else: action = f"Moving it to {invalid_reports_folder} without parsing it" logger.error( f"Message UID {message_id} in {reports_folder} has {count} unfinished " f"parse attempts; the limit is {MAX_PARSE_ATTEMPTS}. {action}" ) return True
[docs] def get_dmarc_reports_from_mbox( input_: str, *, nameservers: list[str] | None = None, dns_timeout: float = DEFAULT_DNS_TIMEOUT, dns_retries: int = DEFAULT_DNS_MAX_RETRIES, strip_attachment_payloads: bool = False, ip_db_path: str | None = None, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, offline: bool = False, normalize_timespan_threshold_hours: float = 24.0, n_procs: int = 1, parse_timeout: float = 0.0, config: ParserConfig | None = None, ) -> ParsingResults: """Parses a mailbox in mbox format containing e-mails with attached DMARC and SMTP TLS reports Args: input_ (str): A path to a mbox file nameservers (list): A list of one or more nameservers to use (Cloudflare's public DNS resolvers by default) dns_timeout (float): Sets the DNS timeout in seconds dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from failure report results always_use_local_files (bool): Do not download files reverse_dns_map_path (str): Path to a reverse DNS map file reverse_dns_map_url (str): URL to a reverse DNS map file ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP offline (bool): Do not make online queries for geolocation or DNS normalize_timespan_threshold_hours (float): Normalize timespans beyond this n_procs (int): Number of processes to use for parsing messages in parallel. Message reading, deduplication, and result assembly stay in the calling process; only parsing is parallelized. Not part of ``config``; always applies. parse_timeout (float): Stop parsing a message after this many seconds and skip it with a warning, as an invalid message (``0``, the default, means no limit). When set, messages are parsed in worker processes even with ``n_procs`` 1. A negative or non-finite value raises ``ValueError``. See ``parsedmarc.parallel.parallel_map()``. Not part of ``config``; always applies. config (ParserConfig): a single object carrying all parsing and enrichment options plus the caches; when provided, the individual option keyword arguments listed above are ignored in favor of the config's values. Returns: dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports`` """ cfg = _resolve_config( config, offline=offline, ip_db_path=ip_db_path, always_use_local_files=always_use_local_files, reverse_dns_map_path=reverse_dns_map_path, reverse_dns_map_url=reverse_dns_map_url, nameservers=nameservers, dns_timeout=dns_timeout, dns_retries=dns_retries, strip_attachment_payloads=strip_attachment_payloads, normalize_timespan_threshold_hours=normalize_timespan_threshold_hours, ) aggregate_reports: list[AggregateReport] = [] failure_reports: list[FailureReport] = [] smtp_tls_reports: list[SMTPTLSReport] = [] try: mbox = mailbox.mbox(input_) message_keys = mbox.keys() total_messages = len(message_keys) logger.debug(f"Found {total_messages} messages in {input_}") if parse_timeout or (n_procs > 1 and total_messages > 1): from parsedmarc.parallel import ( _email_timeout_result, _parse_report_email_job, parallel_map, ) func = functools.partial(_parse_report_email_job, config=cfg) def _jobs(): for i in range(total_messages): message_key = message_keys[i] logger.info(f"Processing message {i + 1} of {total_messages}") yield mbox.get_bytes(message_key) for result in tqdm( parallel_map( func, _jobs(), n_procs, parse_timeout=parse_timeout, timeout_result=_email_timeout_result, ), total=total_messages, disable=None, ): if isinstance(result, (InvalidDMARCReport, InvalidSMTPTLSReport)): logger.warning(str(result)) elif isinstance(result, ParserError): raise result else: _classify_parsed_email( result, aggregate_reports, failure_reports, smtp_tls_reports, seen_aggregate_report_ids=cfg.seen_aggregate_report_ids, ) else: for i in tqdm(range(total_messages), disable=None): message_key = message_keys[i] logger.info(f"Processing message {i + 1} of {total_messages}") msg_content = mbox.get_bytes(message_key) try: parsed_email = parse_report_email(msg_content, config=cfg) _classify_parsed_email( parsed_email, aggregate_reports, failure_reports, smtp_tls_reports, seen_aggregate_report_ids=cfg.seen_aggregate_report_ids, ) except (InvalidDMARCReport, InvalidSMTPTLSReport) as error: logger.warning(error.__str__()) except mailbox.NoSuchMailboxError: raise InvalidDMARCReport(f"Mailbox {input_} does not exist") return { "aggregate_reports": aggregate_reports, "failure_reports": failure_reports, "smtp_tls_reports": smtp_tls_reports, }
def _migrate_forensic_archive_folder( connection: MailboxConnection, archive_folder: str ) -> None: """Consolidate a pre-rename ``<archive>/Forensic`` subfolder into ``<archive>/Failure``. Before failure reports were renamed from "forensic" reports, they were archived under ``<archive_folder>/Forensic``; they now go to ``<archive_folder>/Failure``. This best-effort, run-on-startup migration moves any pre-existing legacy archive into the new location so reports filed before and after the rename live in the same folder. It is a no-op when there is no legacy ``Forensic`` folder (the common case), and never raises: a mailbox that cannot be reorganized is logged and skipped, consistent with the rest of parsedmarc's mailbox handling (warn, don't crash). Uses the folder-management API added in mailsuite 2.1.0 (``folder_exists`` / ``rename_folder`` / ``merge_folders``). """ old_folder = f"{archive_folder}/Forensic" new_folder = f"{archive_folder}/Failure" try: if not connection.folder_exists(old_folder): return if connection.folder_exists(new_folder): # Both exist (e.g. a partial earlier migration, or a manually # created Failure folder): move the legacy folder's messages into # the new one and drop the now-empty legacy folder. connection.merge_folders(old_folder, new_folder) logger.info(f"Merged legacy archive folder {old_folder} into {new_folder}") else: connection.rename_folder(old_folder, new_folder) logger.info(f"Renamed legacy archive folder {old_folder} to {new_folder}") except Exception as error: logger.warning( f"Could not migrate legacy archive folder {old_folder} to {new_folder}: {error}" ) def _ensure_folder(connection: MailboxConnection, folder: str) -> None: """Best-effort create ``folder`` if the backend says it is missing. ``get_dmarc_reports_from_mailbox()`` creates its destination folders up front, but only when ``create_folders`` is set -- watch mode calls it with ``create_folders=False``, and the ``Unsaved`` holding folder is only created up front when a ``save_callback`` was supplied. This is the defensive check immediately before a message is moved there. Like ``_migrate_forensic_archive_folder``, it never raises: a backend that cannot report on or create the folder is logged and skipped, and the move that follows either succeeds anyway (the folder already existed) or fails and is logged by its own error handler -- warn, don't crash. """ try: if not connection.folder_exists(folder): connection.create_folder(folder) except Exception as error: logger.warning(f"Could not create folder {folder}: {error}")
[docs] def get_dmarc_reports_from_mailbox( connection: MailboxConnection, *, reports_folder: str = "INBOX", archive_folder: str = "Archive", delete: bool = False, delete_aggregate: bool | None = None, delete_failure: bool | None = None, delete_smtp_tls: bool | None = None, delete_invalid: bool | None = None, test: bool = False, ip_db_path: str | None = None, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, offline: bool = False, nameservers: list[str] | None = None, dns_timeout: float = DEFAULT_DNS_TIMEOUT, dns_retries: int = DEFAULT_DNS_MAX_RETRIES, strip_attachment_payloads: bool = False, results: ParsingResults | None = None, batch_size: int = 10, since: datetime | date | str | None = None, create_folders: bool = True, normalize_timespan_threshold_hours: float = 24.0, n_procs: int = 1, parse_timeout: float = 0.0, save_callback: Callable[[ParsingResults], bool | None] | None = None, max_unsaved_retries: int = 2, attempts_file: str | None = None, config: ParserConfig | None = None, ) -> ParsingResults: """ Fetches and parses DMARC and SMTP TLS reports from a mailbox Args: connection: A Mailbox connection object reports_folder (str): The folder where reports can be found archive_folder (str): The folder to move processed mail to delete (bool): Delete messages after processing them delete_aggregate (bool | None): Delete aggregate report messages after processing them, instead of moving them to the ``Aggregate`` archive subfolder; ``None`` (the default) inherits the value of ``delete`` delete_failure (bool | None): Delete failure report messages after processing them, instead of moving them to the ``Failure`` archive subfolder; ``None`` (the default) inherits the value of ``delete`` delete_smtp_tls (bool | None): Delete SMTP TLS report messages after processing them, instead of moving them to the ``SMTP-TLS`` archive subfolder; ``None`` (the default) inherits the value of ``delete`` delete_invalid (bool | None): Delete unparseable messages, instead of moving them to the ``Invalid`` archive subfolder where they can be inspected for debugging; ``None`` (the default) inherits the value of ``delete`` test (bool): Do not move or delete messages after processing them ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP always_use_local_files (bool): Do not download files reverse_dns_map_path (str): Path to a reverse DNS map file reverse_dns_map_url (str): URL to a reverse DNS map file offline (bool): Do not query online for geolocation or DNS nameservers (list): A list of DNS nameservers to query dns_timeout (float): Set the DNS query timeout dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from failure report results results (dict): Results from the previous run batch_size (int): Number of messages to read and process before saving (use 0 for no limit) since: Search for messages since a certain time (units - {"m":"minutes", "h":"hours", "d":"days", "w":"weeks"}) create_folders (bool): Whether to create the destination folders (not used in watch) normalize_timespan_threshold_hours (float): Normalize timespans beyond this n_procs (int): Number of processes to use for parsing messages in parallel. Fetching, archiving, and deduplication remain sequential in the calling process; only parsing is parallelized. With ``n_procs > 1``, invalid-message disposition happens after the parsing phase completes, rather than interleaved message-by-message as it is when ``n_procs`` is 1. Not part of ``config``; always applies. parse_timeout (float): Stop parsing a message after this many seconds and move it to the ``Invalid`` archive subfolder, as an invalid message (``0``, the default, means no limit). When set, messages are parsed in worker processes even with ``n_procs`` 1, so disposition of invalid messages happens after the parsing phase as it does with ``n_procs > 1``. A negative or non-finite value raises ``ValueError``. See ``parsedmarc.parallel.parallel_map()``. Not part of ``config``; always applies. save_callback: An optional callable invoked once per fetched batch with a ``ParsingResults`` dict holding only that batch's newly parsed reports, after parsing but before any of the batch's messages are deleted or moved out of ``reports_folder``. It tells this function whether the batch was actually persisted: * Returning ``False`` means "not saved": the batch's messages are left in ``reports_folder`` for retry on the next run (or moved to ``{archive_folder}/Unsaved`` once they have failed ``max_unsaved_retries`` retries -- see below), and the aggregate-report dedup cache is not updated for that batch, so a retry reparses the same reports instead of skipping them as duplicates. * Raising counts as "not saved" too: the same bookkeeping runs (the batch's messages are held back or moved to ``Unsaved`` at the cap, and the failed attempt counts toward ``max_unsaved_retries``), and the exception is then re-raised to the caller. * Any other return value, including ``None``, commits the batch: the dedup cache is updated and the messages are deleted or archived exactly as they are with no callback. ``None`` (the default) commits every batch, preserving the prior behavior. The callback is still invoked when ``test`` is ``True``, so a test run exercises the full pipeline, but neither the mailbox nor the retry counters are touched regardless of what it returns. max_unsaved_retries (int): How many times a message may be *retried* after ``save_callback`` first reported its batch unsaved, before it is moved to the ``Unsaved`` archive subfolder instead of being retried again (default 2, i.e. the initial attempt plus two retries -- at most three deliveries to any output destination that does not deduplicate). ``0`` moves a message on the first failed save; negative values raise ``ValueError``. Messages moved to ``Unsaved`` are never deleted, whatever the ``delete`` options say; recover them by fixing the output destination and moving them back into ``reports_folder``. Counts are kept in memory, per message and per process, are reset by a successful save, and are only kept when a ``save_callback`` is supplied -- so the cap applies across repeated calls within one process (``watch_inbox()``'s checks), not across separate one-shot processes, each of which starts a message's count over. Mailbox orchestration, so like ``batch_size`` it is not part of ``config``. attempts_file (str | None): A JSON file in which to count, per message, the parse attempts that started but never finished, because the process died while parsing (the out-of-memory killer, a container memory cap). An attempt is recorded before a message is parsed and cleared as soon as the parse returns a report or raises ``ParserError``, so a failed save is not counted. A message that has died ``MAX_PARSE_ATTEMPTS`` (2) times is logged at error level and disposed of as invalid (moved to ``Invalid``, or deleted under ``delete_invalid``) without being parsed again. With ``n_procs > 1`` or ``parse_timeout`` set, a message that was being parsed when an earlier run died is parsed on its own after the others (in this process, or with ``parse_timeout`` in a worker pool of its own, so the timeout still applies), so the messages parsed beside it are not blamed for it. ``None`` (the default) keeps no record. Mailbox orchestration, not part of ``config``. config (ParserConfig): a single object carrying all parsing and enrichment options plus the caches; when provided, it replaces the individual parsing and enrichment option keyword arguments listed above (DNS, GeoIP, offline mode, attachment payload stripping, timespan normalization), whose values are then ignored. The remaining keyword arguments control mailbox handling and orchestration rather than parsing (the folder names, the ``delete`` options, ``test``, ``since``, ``batch_size``, ``save_callback``, ``max_unsaved_retries``, ``attempts_file``); they are not part of ``config`` and always apply. Returns: dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports`` """ # Each per-report-type flag inherits the overall ``delete`` value when it # is left unset (``None``). Resolve once, up front: every decision below # reads only these four resolved flags. The raw ``delete`` parameter is # still forwarded verbatim to the recursive self-call at the end of this # function, where it is inert because all four resolved flags accompany # it. delete_aggregate = delete if delete_aggregate is None else delete_aggregate delete_failure = delete if delete_failure is None else delete_failure delete_smtp_tls = delete if delete_smtp_tls is None else delete_smtp_tls delete_invalid = delete if delete_invalid is None else delete_invalid if test and ( delete_aggregate or delete_failure or delete_smtp_tls or delete_invalid ): raise ValueError("delete options and test are mutually exclusive") if max_unsaved_retries < 0: raise ValueError("max_unsaved_retries must be >= 0") from parsedmarc.parallel import _check_parse_timeout _check_parse_timeout(parse_timeout) if connection is None: raise ValueError("Must supply a connection") attempt_store = AttemptStore(attempts_file) if attempts_file else None cfg = _resolve_config( config, offline=offline, ip_db_path=ip_db_path, always_use_local_files=always_use_local_files, reverse_dns_map_path=reverse_dns_map_path, reverse_dns_map_url=reverse_dns_map_url, nameservers=nameservers, dns_timeout=dns_timeout, dns_retries=dns_retries, strip_attachment_payloads=strip_attachment_payloads, normalize_timespan_threshold_hours=normalize_timespan_threshold_hours, ) # current_time useful to fetch_messages later in the program current_time: datetime | date | str | None = None aggregate_reports: list[AggregateReport] = [] failure_reports: list[FailureReport] = [] smtp_tls_reports: list[SMTPTLSReport] = [] # This call's own reports, kept separate from the accumulated lists # above (which carry earlier batches' reports in via ``results``) so the # save callback is handed only what this batch parsed. batch_aggregate_reports: list[AggregateReport] = [] batch_failure_reports: list[FailureReport] = [] batch_smtp_tls_reports: list[SMTPTLSReport] = [] # Aggregate dedup keys are staged here and only written to the shared # cache once the batch is known to be saved, so an unsaved batch (or a # mid-batch crash) leaves the cache clean and the reports are reparsed # on the retry instead of being dropped as duplicates. pending_aggregate_keys: set[str] = set() aggregate_report_msg_uids = [] failure_report_msg_uids = [] smtp_tls_msg_uids = [] aggregate_reports_folder = f"{archive_folder}/Aggregate" failure_reports_folder = f"{archive_folder}/Failure" smtp_tls_reports_folder = f"{archive_folder}/SMTP-TLS" invalid_reports_folder = f"{archive_folder}/Invalid" unsaved_reports_folder = f"{archive_folder}/Unsaved" if results: aggregate_reports = results["aggregate_reports"].copy() failure_reports = results["failure_reports"].copy() smtp_tls_reports = results["smtp_tls_reports"].copy() if not test and create_folders: _migrate_forensic_archive_folder(connection, archive_folder) connection.create_folder(archive_folder) connection.create_folder(aggregate_reports_folder) connection.create_folder(failure_reports_folder) connection.create_folder(smtp_tls_reports_folder) connection.create_folder(invalid_reports_folder) if save_callback is not None: # Only reachable when a callback can report a batch unsaved; # without one no message is ever held back, so the folder would # sit empty in every mailbox. connection.create_folder(unsaved_reports_folder) if since and isinstance(since, str): _since = 1440 # default one day if re.match(r"\d+[mhdw]$", since): s = re.split(r"(\d+)", since) if s[2] == "m": _since = int(s[1]) elif s[2] == "h": _since = int(s[1]) * 60 elif s[2] == "d": _since = int(s[1]) * 60 * 24 elif s[2] == "w": _since = int(s[1]) * 60 * 24 * 7 else: logger.warning( f"Incorrect format for 'since' option. Provided value: {since}, " "expected values: (5m|3h|2d|1w). Ignoring option, fetching " "messages for last 24hrs. SMTP does not support a time or " "timezone in since. See " "https://www.rfc-editor.org/rfc/rfc3501#page-52" ) if isinstance(connection, IMAPConnection): logger.debug( "Only days and weeks values in 'since' option are considered " "for IMAP connections. Examples: 2d or 1w" ) since = (datetime.now(timezone.utc) - timedelta(minutes=_since)).strftime( "%d-%b-%Y" ) current_time = datetime.now(timezone.utc).strftime("%d-%b-%Y") elif isinstance(connection, MSGraphConnection): since = (datetime.now(timezone.utc) - timedelta(minutes=_since)).isoformat() current_time = datetime.now(timezone.utc).isoformat() elif isinstance(connection, GmailConnection): since = (datetime.now(timezone.utc) - timedelta(minutes=_since)).strftime( "%s" ) current_time = datetime.now(timezone.utc).strftime("%s") else: pass messages = connection.fetch_messages( reports_folder, batch_size=batch_size, since=since ) total_messages = len(messages) logger.debug(f"Found {len(messages)} messages in {reports_folder}") if batch_size and not since: message_limit = min(total_messages, batch_size) else: message_limit = total_messages logger.debug(f"Processing {message_limit} messages") def _parse_finished(key: str | None) -> None: # A parse that returned a result or raised ParserError did not # crash the process, so its attempt is cleared before anything that # can fail for other reasons (the save, the move) runs. if attempt_store is not None and key is not None: attempt_store.clear(key) def _parse_in_process( message_id: int | str, msg_content: str | ParserError ) -> None: key: str | None = None try: if isinstance(msg_content, ParserError): raise msg_content if attempt_store is not None: key = attempt_key(reports_folder, message_id, msg_content) if _record_parse_attempt( attempt_store, key, message_id, reports_folder, delete_invalid, invalid_reports_folder, test, ): # The entry is kept until the message has left the # folder, so a failed move is retried on the next run. if not test and _dispose_invalid_message( connection, message_id, delete_invalid, invalid_reports_folder ): attempt_store.clear(key) return parsed_email = parse_report_email( msg_content, config=cfg, keep_alive=connection.keepalive, ) _parse_finished(key) report_type = _classify_parsed_email( parsed_email, batch_aggregate_reports, batch_failure_reports, batch_smtp_tls_reports, seen_aggregate_report_ids=cfg.seen_aggregate_report_ids, pending_aggregate_keys=pending_aggregate_keys, ) if report_type == "aggregate": aggregate_report_msg_uids.append(message_id) elif report_type == "failure": failure_report_msg_uids.append(message_id) elif report_type == "smtp_tls": smtp_tls_msg_uids.append(message_id) except ParserError as error: _parse_finished(key) logger.warning(error.__str__()) if not test: _dispose_invalid_message( connection, message_id, delete_invalid, invalid_reports_folder ) if parse_timeout or (n_procs > 1 and message_limit > 1): from parsedmarc.parallel import ( _email_timeout_result, _parse_report_email_job, parallel_map, ) # The config's caches (ip_address_cache, seen_aggregate_report_ids, # reverse_dns_map) never cross the process boundary -- # ParserConfig.__getstate__ drops them, and each worker accumulates # its own via the module defaults it rebinds to on unpickling (see # ParserConfig.__setstate__). keep_alive is a bound method of the # live connection object and is not a ParserConfig field, so it is # never submitted to the pool either; the heartbeat passed to # parallel_map below keeps the connection alive instead. func = functools.partial(_parse_report_email_job, config=cfg) # parallel_map yields results in submission order, so the oldest # queued id always belongs to the next yielded result; popping as # results arrive keeps this queue no larger than the in-flight # submission window. Each id travels with its attempts-file key # (None when there is no attempts file, or no attempt to clear). fetched_ids: deque[tuple[int | str, str | None]] = deque() invalid_msg_ids: list[tuple[int | str, str | None]] = [] # Messages that were being parsed when an earlier run died. That # run had up to a whole submission window in flight (with n_procs # above 1, or with parse_timeout set, which takes this path even # with one process), so all of them are suspects; parsing each alone, after the others, lets the next # crash count against the one message that caused it. suspects: list[tuple[int | str, str]] = [] def _jobs(): for i in range(message_limit): msg_uid = messages[i] logger.debug( f"Processing message {i + 1} of {message_limit}: UID {msg_uid}" ) message_id, msg_content = _fetch_mailbox_message( connection, msg_uid, test ) if isinstance(msg_content, ParserError): # Never submitted, so never yielded and never queued in # fetched_ids; the post-loop disposal handles it. logger.warning(str(msg_content)) invalid_msg_ids.append((message_id, None)) continue key: str | None = None if attempt_store is not None: key = attempt_key(reports_folder, message_id, msg_content) if 0 < attempt_store.count(key) < MAX_PARSE_ATTEMPTS: logger.warning( f"Message UID {message_id} in {reports_folder} " "was being parsed when parsedmarc last stopped " "without finishing; parsing it on its own after " "the rest of the batch" ) suspects.append((message_id, msg_content)) continue if _record_parse_attempt( attempt_store, key, message_id, reports_folder, delete_invalid, invalid_reports_folder, test, ): invalid_msg_ids.append((message_id, key)) continue fetched_ids.append((message_id, key)) yield msg_content def _take_result( message_id: int | str, key: str | None, result: ParsedReport | ParserError, ) -> None: _parse_finished(key) if isinstance(result, ParserError): logger.warning(str(result)) invalid_msg_ids.append((message_id, None)) else: report_type = _classify_parsed_email( result, batch_aggregate_reports, batch_failure_reports, batch_smtp_tls_reports, seen_aggregate_report_ids=cfg.seen_aggregate_report_ids, pending_aggregate_keys=pending_aggregate_keys, ) if report_type == "aggregate": aggregate_report_msg_uids.append(message_id) elif report_type == "failure": failure_report_msg_uids.append(message_id) elif report_type == "smtp_tls": smtp_tls_msg_uids.append(message_id) for result in parallel_map( func, _jobs(), n_procs, heartbeat=connection.keepalive, parse_timeout=parse_timeout, timeout_result=_email_timeout_result, ): message_id, key = fetched_ids.popleft() _take_result(message_id, key, result) for message_id, msg_content in suspects: if not parse_timeout or attempt_store is None: _parse_in_process(message_id, msg_content) continue # Only a worker process can be stopped when the timeout expires, # so each suspect gets a pool of its own: the timeout still # bounds it, and a crash is still charged to it alone. A suspect # is below MAX_PARSE_ATTEMPTS, so this record cannot reach the # cap. key = attempt_key(reports_folder, message_id, msg_content) attempt_store.record(key) for result in parallel_map( func, [msg_content], 1, heartbeat=connection.keepalive, parse_timeout=parse_timeout, timeout_result=_email_timeout_result, ): _take_result(message_id, key, result) if not test: for invalid_message_id, key in invalid_msg_ids: # A set-aside message's entry is cleared only once it has # left the folder, as in _parse_in_process. if _dispose_invalid_message( connection, invalid_message_id, delete_invalid, invalid_reports_folder, ): _parse_finished(key) else: for i in range(message_limit): msg_uid = messages[i] logger.debug( f"Processing message {i + 1} of {message_limit}: UID {msg_uid}" ) message_id, msg_content = _fetch_mailbox_message(connection, msg_uid, test) _parse_in_process(message_id, msg_content) # Ask the caller whether this batch actually made it to its output # destinations before touching a single message. A callback that says # otherwise (or raises) means the reports exist nowhere else yet, so no # message may be archived or deleted -- only retained for retry, or moved # intact to the Unsaved folder once retries run out (#242). batch_results: ParsingResults = { "aggregate_reports": batch_aggregate_reports, "failure_reports": batch_failure_reports, "smtp_tls_reports": batch_smtp_tls_reports, } persisted = True callback_error: Exception | None = None if save_callback is not None: try: persisted = save_callback(batch_results) is not False except Exception as error: # A raising callback is a failed save too: the failure # bookkeeping below still runs (count the attempt, hold or shelve # the messages) before the exception is re-raised, so a callback # that always raises -- e.g. the CLI's under # ``fail_on_output_error`` -- is still bounded by # ``max_unsaved_retries`` even when the caller's watch loop # swallows the exception and keeps checking, as mailsuite's IMAP # and Maildir backends do. persisted = False callback_error = error batch_msg_uids = ( aggregate_report_msg_uids + failure_report_msg_uids + smtp_tls_msg_uids ) if persisted: # Committing: the reports are safely stored elsewhere, so record # their dedup keys and forget any earlier failures for these # messages. for report_key in pending_aggregate_keys: cfg.seen_aggregate_report_ids[report_key] = True if not test: for msg_uid in batch_msg_uids: _FAILED_SAVE_ATTEMPTS.pop((reports_folder, str(msg_uid)), None) elif not test: # Held back for retry. Messages that have now failed the initial # attempt plus ``max_unsaved_retries`` retries stop being retried and # move to the Unsaved folder, bounding duplicate delivery to output # destinations that do not deduplicate; the rest stay put. retained_uids: list[int | str] = [] over_cap_uids: list[int | str] = [] highest_attempt = 0 for msg_uid in batch_msg_uids: attempts_key = (reports_folder, str(msg_uid)) attempts = _FAILED_SAVE_ATTEMPTS.get(attempts_key, 0) + 1 _FAILED_SAVE_ATTEMPTS[attempts_key] = attempts if attempts > max_unsaved_retries: over_cap_uids.append(msg_uid) else: retained_uids.append(msg_uid) highest_attempt = max(highest_attempt, attempts) if retained_uids: logger.error( f"Reports were not saved: leaving {len(retained_uids)} " f"message(s) in {reports_folder} to retry on the next run " f"or check (failed attempt {highest_attempt} of " f"{max_unsaved_retries + 1})" ) if over_cap_uids: logger.error( f"Reports were not saved after {max_unsaved_retries + 1} " f"attempt(s): moving {len(over_cap_uids)} message(s) from " f"{reports_folder} to {unsaved_reports_folder} instead of " "retrying them further. They are never deleted -- fix the " f"output destination, then move them back to {reports_folder}" ) _ensure_folder(connection, unsaved_reports_folder) for msg_uid in over_cap_uids: try: connection.move_message(msg_uid, unsaved_reports_folder) except Exception as e: e = f"Error moving message UID {msg_uid}: {e}" logger.error(f"Mailbox error: {e}") else: # Drop the counter only once the message is actually out # of the retry loop. Clearing it before a failed move # would hand the still-in-place message a fresh set of # under-cap retries (and deliveries); keeping it means # the next failed save classifies the message over-cap # again and re-attempts the move instead. _FAILED_SAVE_ATTEMPTS.pop((reports_folder, str(msg_uid)), None) if callback_error is not None: raise callback_error aggregate_reports += batch_aggregate_reports failure_reports += batch_failure_reports smtp_tls_reports += batch_smtp_tls_reports if persisted and not test: # Each report type is disposed of according to its own effective # delete flag: deleted outright, or moved to its archive subfolder. for msg_uids, delete_type, destination_folder, label in ( ( aggregate_report_msg_uids, delete_aggregate, aggregate_reports_folder, "aggregate report", ), ( failure_report_msg_uids, delete_failure, failure_reports_folder, "failure report", ), ( smtp_tls_msg_uids, delete_smtp_tls, smtp_tls_reports_folder, "SMTP TLS report", ), ): number_of_msgs = len(msg_uids) if number_of_msgs == 0: continue if not delete_type: message = f"Moving {label} messages from" logger.debug(f"{message} {reports_folder} to {destination_folder}") for i in range(number_of_msgs): msg_uid = msg_uids[i] if delete_type: logger.debug( f"Deleting message {i + 1} of {number_of_msgs}: UID {msg_uid}" ) try: connection.delete_message(msg_uid) except Exception as e: message = "Error deleting message UID" e = f"{message} {msg_uid}: {e}" logger.error(f"Mailbox error: {e}") else: message = "Moving message" logger.debug( f"{message} {i + 1} of {number_of_msgs}: UID {msg_uid}" ) try: connection.move_message(msg_uid, destination_folder) except Exception as e: e = f"Error moving message UID {msg_uid}: {e}" logger.error(f"Mailbox error: {e}") results = { "aggregate_reports": aggregate_reports, "failure_reports": failure_reports, "smtp_tls_reports": smtp_tls_reports, } # An unsaved batch left its messages in ``reports_folder``, so the # re-check below would find them again and immediately reprocess the # very messages that just failed -- burning through the retry cap in one # call. Skip it and let the next run retry them. if persisted and not test and not batch_size: if current_time: total_messages = len( connection.fetch_messages(reports_folder, since=current_time) ) else: total_messages = len(connection.fetch_messages(reports_folder)) else: total_messages = 0 if total_messages > 0: # Process emails that came in during the last run results = get_dmarc_reports_from_mailbox( connection=connection, reports_folder=reports_folder, archive_folder=archive_folder, delete=delete, delete_aggregate=delete_aggregate, delete_failure=delete_failure, delete_smtp_tls=delete_smtp_tls, delete_invalid=delete_invalid, test=test, results=results, since=current_time, n_procs=n_procs, parse_timeout=parse_timeout, save_callback=save_callback, max_unsaved_retries=max_unsaved_retries, attempts_file=attempts_file, config=cfg, ) return results
[docs] def watch_inbox( mailbox_connection: MailboxConnection, callback: Callable, *, reports_folder: str = "INBOX", archive_folder: str = "Archive", delete: bool = False, delete_aggregate: bool | None = None, delete_failure: bool | None = None, delete_smtp_tls: bool | None = None, delete_invalid: bool | None = None, test: bool = False, check_timeout: int = 30, ip_db_path: str | None = None, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, offline: bool = False, nameservers: list[str] | None = None, dns_timeout: float = DEFAULT_DNS_TIMEOUT, dns_retries: int = DEFAULT_DNS_MAX_RETRIES, strip_attachment_payloads: bool = False, batch_size: int = 10, since: datetime | date | str | None = None, normalize_timespan_threshold_hours: float = 24.0, config_reloading: Callable | None = None, n_procs: int = 1, parse_timeout: float = 0.0, max_unsaved_retries: int = 2, attempts_file: str | None = None, config: ParserConfig | None = None, ): """ Watches the mailbox for new messages and sends the results to a callback function Args: mailbox_connection: The mailbox connection object callback: The callback function to receive the parsing results. Passed straight through to ``get_dmarc_reports_from_mailbox()`` as its ``save_callback``, so it now runs once per fetched batch, with only that batch's reports, *before* those messages are deleted or moved out of ``reports_folder`` -- rather than once afterward with the whole check's accumulated results. Returning ``False`` reports the batch as unsaved, leaving its messages in place to be retried on the next check instead of archived or deleted (see ``save_callback`` and ``max_unsaved_retries`` on ``get_dmarc_reports_from_mailbox()``). Raising counts as an unsaved batch too -- same retention and retry cap -- before the exception reaches the mailbox backend's watch loop; what happens then is backend-specific: the Microsoft Graph and Gmail backends let it propagate and end the watch, while mailsuite's IMAP and Maildir watch loops log it and keep checking. reports_folder (str): The folder where reports can be found archive_folder (str): The folder to move processed mail to delete (bool): Delete messages after processing them delete_aggregate (bool | None): Delete aggregate report messages after processing them, instead of moving them to the ``Aggregate`` archive subfolder; ``None`` (the default) inherits the value of ``delete`` delete_failure (bool | None): Delete failure report messages after processing them, instead of moving them to the ``Failure`` archive subfolder; ``None`` (the default) inherits the value of ``delete`` delete_smtp_tls (bool | None): Delete SMTP TLS report messages after processing them, instead of moving them to the ``SMTP-TLS`` archive subfolder; ``None`` (the default) inherits the value of ``delete`` delete_invalid (bool | None): Delete unparseable messages, instead of moving them to the ``Invalid`` archive subfolder where they can be inspected for debugging; ``None`` (the default) inherits the value of ``delete`` test (bool): Do not move or delete messages after processing them check_timeout (int): Number of seconds to wait for an IMAP IDLE response or the number of seconds until the next mail check ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP always_use_local_files (bool): Do not download files reverse_dns_map_path (str): Path to a reverse DNS map file reverse_dns_map_url (str): URL to a reverse DNS map file offline (bool): Do not query online for geolocation or DNS nameservers (list): A list of one or more nameservers to use (Cloudflare's public DNS resolvers by default) dns_timeout (float): Set the DNS query timeout dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from failure report results batch_size (int): Number of messages to read and process before saving since: Search for messages since a certain time normalize_timespan_threshold_hours (float): Normalize timespans beyond this config_reloading: Optional callable that returns True when a config reload (or shutdown) has been requested (e.g. via SIGHUP/SIGTERM). Polled by the mailbox backend between checks, including the IMAP IDLE loop, so the watcher exits cleanly at a safe boundary. n_procs (int): Number of processes to use for parsing messages in parallel. Passed through to ``get_dmarc_reports_from_mailbox`` on each check. Not part of ``config``; always applies. parse_timeout (float): Stop parsing a message after this many seconds and treat it as invalid (``0``, the default, means no limit). Passed through to ``get_dmarc_reports_from_mailbox`` on each check; a negative or non-finite value raises ``ValueError`` before the watch starts. Not part of ``config``; always applies. max_unsaved_retries (int): How many times a message may be retried after ``callback`` first reported its batch unsaved, before it is moved to the ``Unsaved`` archive subfolder (default 2). Passed through to ``get_dmarc_reports_from_mailbox``, where it is documented in full. Not part of ``config``; always applies. attempts_file (str | None): A JSON file counting the parse attempts that never finished, so a message that keeps killing the process is set aside as invalid (default ``None``: no record). Passed through to ``get_dmarc_reports_from_mailbox``, where it is documented in full. Not part of ``config``; always applies. config (ParserConfig): a single object carrying all parsing and enrichment options plus the caches; when provided, it replaces the individual parsing and enrichment option keyword arguments listed above (DNS, GeoIP, offline mode, attachment payload stripping, timespan normalization), whose values are then ignored. The remaining keyword arguments control mailbox handling and orchestration rather than parsing (the folder names, the ``delete`` options, ``test``, ``since``, ``batch_size``, ``max_unsaved_retries``, ``attempts_file``); they are not part of ``config`` and always apply. """ # Validate before the watch loop starts: raised inside a check, this # would be swallowed and endlessly retried by the IMAP and Maildir # backends' per-check exception handling instead of surfacing. if max_unsaved_retries < 0: raise ValueError("max_unsaved_retries must be >= 0") from parsedmarc.parallel import _check_parse_timeout _check_parse_timeout(parse_timeout) cfg = _resolve_config( config, offline=offline, ip_db_path=ip_db_path, always_use_local_files=always_use_local_files, reverse_dns_map_path=reverse_dns_map_path, reverse_dns_map_url=reverse_dns_map_url, nameservers=nameservers, dns_timeout=dns_timeout, dns_retries=dns_retries, strip_attachment_payloads=strip_attachment_payloads, normalize_timespan_threshold_hours=normalize_timespan_threshold_hours, ) def check_callback(connection): get_dmarc_reports_from_mailbox( connection=connection, reports_folder=reports_folder, archive_folder=archive_folder, delete=delete, delete_aggregate=delete_aggregate, delete_failure=delete_failure, delete_smtp_tls=delete_smtp_tls, delete_invalid=delete_invalid, test=test, batch_size=batch_size, since=since, create_folders=False, n_procs=n_procs, parse_timeout=parse_timeout, save_callback=callback, max_unsaved_retries=max_unsaved_retries, attempts_file=attempts_file, config=cfg, ) watch_kwargs: dict = { "check_callback": check_callback, "check_timeout": check_timeout, } if config_reloading is not None: watch_kwargs["config_reloading"] = config_reloading mailbox_connection.watch(**watch_kwargs)
[docs] def append_json( filename: str, reports: Sequence[AggregateReport] | Sequence[FailureReport] | Sequence[SMTPTLSReport], ) -> None: """Append ``reports`` to a JSON array on disk, creating the file if needed. Reads the existing array (if the file exists and parses cleanly), merges the new reports onto the end, and rewrites the file as a single valid JSON array. An earlier version of this used an ``open(..., "a+")`` + ``seek()`` + overwrite pattern, but Python's documentation is explicit that on POSIX, ``a`` / ``a+`` writes *always* go to EOF regardless of seek position — so the second call onto an existing file produced ``[...],\\n[...]``-style corrupted output. Read-merge-write is the only way to get a valid JSON array out of repeated appends. """ if len(reports) == 0: # Don't create an empty-array file for an empty input; if a # file already exists, leave it alone. return existing: list = [] if os.path.isfile(filename) and os.path.getsize(filename) > 0: try: with open(filename, "r", encoding="utf-8") as f: loaded = json.loads(f.read()) if isinstance(loaded, list): existing = loaded except (json.JSONDecodeError, OSError): # Corrupted or unreadable: overwrite cleanly rather than # silently fail to record. existing = [] merged = existing + list(reports) with open(filename, "w", newline="\n", encoding="utf-8") as output: json.dump(merged, output, ensure_ascii=False, indent=2)
def append_csv(filename: str, csv: str) -> None: with open(filename, "a+", newline="\n", encoding="utf-8") as output: if output.seek(0, os.SEEK_END) != 0: # strip the headers from the CSV _headers, csv = csv.split("\n", 1) if len(csv) == 0: # not appending anything, don't do any dance to # append it correctly return output.write(csv)
[docs] def save_output( results: ParsingResults, *, output_directory: str = "output", aggregate_json_filename: str = "aggregate.json", failure_json_filename: str = "failure.json", smtp_tls_json_filename: str = "smtp_tls.json", aggregate_csv_filename: str = "aggregate.csv", failure_csv_filename: str = "failure.csv", smtp_tls_csv_filename: str = "smtp_tls.csv", ): """ Save report data in the given directory The message sample of each failure report is written to a ``samples`` subdirectory, named after the sample's own ``subject`` header run through :func:`parsedmarc.utils.get_filename_safe_string`, falling back to ``sample`` when sanitizing leaves nothing. Since a subject arrives from an untrusted sender, sanitizing happens here, at write time; the ``filename_safe_subject`` key a caller may supply alongside it is not trusted and not used. Names that collide get a ``(1)``, ``(2)``, … suffix. Args: results: Parsing results output_directory (str): The path to the directory to save in aggregate_json_filename (str): Filename for the aggregate JSON file failure_json_filename (str): Filename for the failure JSON file smtp_tls_json_filename (str): Filename for the SMTP TLS JSON file aggregate_csv_filename (str): Filename for the aggregate CSV file failure_csv_filename (str): Filename for the failure CSV file smtp_tls_csv_filename (str): Filename for the SMTP TLS CSV file """ aggregate_reports = results["aggregate_reports"] failure_reports = results["failure_reports"] smtp_tls_reports = results["smtp_tls_reports"] output_directory = os.path.expanduser(output_directory) if os.path.exists(output_directory): if not os.path.isdir(output_directory): raise ValueError(f"{output_directory} is not a directory") else: os.makedirs(output_directory) append_json( os.path.join(output_directory, aggregate_json_filename), aggregate_reports ) append_csv( os.path.join(output_directory, aggregate_csv_filename), parsed_aggregate_reports_to_csv(aggregate_reports), ) append_json(os.path.join(output_directory, failure_json_filename), failure_reports) append_csv( os.path.join(output_directory, failure_csv_filename), parsed_failure_reports_to_csv(failure_reports), ) append_json( os.path.join(output_directory, smtp_tls_json_filename), smtp_tls_reports ) append_csv( os.path.join(output_directory, smtp_tls_csv_filename), parsed_smtp_tls_reports_to_csv(smtp_tls_reports), ) samples_directory = os.path.join(output_directory, "samples") if not os.path.exists(samples_directory): os.makedirs(samples_directory) sample_filenames = [] for failure_report in failure_reports: sample = failure_report["sample"] message_count = 0 parsed_sample = failure_report["parsed_sample"] subject = get_filename_safe_string(parsed_sample.get("subject")) or "sample" filename = subject while filename in sample_filenames: message_count += 1 filename = f"{subject} ({message_count})" sample_filenames.append(filename) filename = f"{filename}.eml" path = os.path.join(samples_directory, filename) with open(path, "w", newline="\n", encoding="utf-8") as sample_file: sample_file.write(sample)
[docs] def get_report_zip(results: ParsingResults) -> bytes: """ Creates a zip file of parsed report output Args: results: The parsed results Returns: bytes: zip file bytes """ def add_subdir(root_path, subdir): subdir_path = os.path.join(root_path, subdir) for subdir_root, subdir_dirs, subdir_files in os.walk(subdir_path): for subdir_file in subdir_files: subdir_file_path = os.path.join(root_path, subdir, subdir_file) if os.path.isfile(subdir_file_path): rel_path = os.path.relpath(subdir_root, subdir_file_path) subdir_arc_name = os.path.join(rel_path, subdir_file) zip_file.write(subdir_file_path, subdir_arc_name) for subdir in subdir_dirs: add_subdir(subdir_path, subdir) storage = BytesIO() tmp_dir = tempfile.mkdtemp() try: save_output(results, output_directory=tmp_dir) with zipfile.ZipFile(storage, "w", zipfile.ZIP_DEFLATED) as zip_file: for root, dirs, files in os.walk(tmp_dir): for file in files: file_path = os.path.join(root, file) if os.path.isfile(file_path): arcname = os.path.join(os.path.relpath(root, tmp_dir), file) zip_file.write(file_path, arcname) for directory in dirs: dir_path = os.path.join(root, directory) if os.path.isdir(dir_path): zip_file.write(dir_path, directory) add_subdir(root, directory) finally: shutil.rmtree(tmp_dir) return storage.getvalue()
def _build_report_email_content( results: ParsingResults, *, subject: str | None = None, attachment_filename: str | None = None, message: str | None = None, ) -> tuple[str, str, list[tuple[str, bytes]]]: """Builds the subject, plain-text body, and zip attachment shared by every report-summary email transport. Returns: A ``(subject, plain_message, attachments)`` tuple. """ date_string = datetime.now().strftime("%Y-%m-%d") if attachment_filename: if not attachment_filename.lower().endswith(".zip"): attachment_filename += ".zip" filename = attachment_filename else: filename = f"DMARC-{date_string}.zip" if subject is None: subject = f"DMARC results for {date_string}" if message is None: message = f"DMARC results for {date_string}" zip_bytes = get_report_zip(results) attachments = [(filename, zip_bytes)] return subject, message, attachments
[docs] def email_results( results: ParsingResults, host: str, mail_from: str, mail_to: list[str] | None, *, mail_cc: list[str] | None = None, mail_bcc: list[str] | None = None, port: int = 0, require_encryption: bool = False, verify: bool = True, username: str | None = None, password: str | None = None, subject: str | None = None, attachment_filename: str | None = None, message: str | None = None, starttls: bool | None = None, ): """ Emails parsing results as a zip file Args: results (dict): Parsing results host (str): Mail server hostname or IP address mail_from: The value of the message from header mail_to (list): A list of addresses to mail to mail_cc (list): A list of addresses to CC mail_bcc (list): A list of addresses to BCC port (int): Port to use require_encryption (bool): Connect with implicit SSL/TLS from the start (usually port 465). When ``False``, the connection starts in plain text and ``starttls`` decides whether it is upgraded. verify (bool): verify the SSL/TLS certificate username (str): An optional username password (str): An optional password subject (str): Overrides the default message subject attachment_filename (str): Override the default attachment filename message (str): Override the default plain text body starttls (bool | None): Whether to upgrade a plain-text (``require_encryption=False``) connection with STARTTLS. ``None`` (the default) upgrades when the server advertises STARTTLS; when it does not, ``mailsuite.smtp.SMTPError`` is raised before authenticating if ``username`` and ``password`` are given, so credentials are never sent in plain text, and without credentials the message is sent in plain text and a warning is logged. ``True`` requires the upgrade and raises ``mailsuite.smtp.SMTPError`` if the server does not advertise STARTTLS. ``False`` never upgrades, even when the server advertises STARTTLS, so any credentials and the message are sent in plain text (a warning is logged when credentials are sent). Ignored when ``require_encryption=True``. """ logger.debug("Emailing report") assert isinstance(mail_to, list) subject, message, attachments = _build_report_email_content( results, subject=subject, attachment_filename=attachment_filename, message=message, ) send_email( host, mail_from, mail_to, message_cc=mail_cc, message_bcc=mail_bcc, port=port, require_encryption=require_encryption, verify=verify, username=username, password=password, subject=subject, attachments=attachments, plain_message=message, starttls=starttls, )
[docs] def email_results_via_msgraph( results: ParsingResults, connection: MSGraphConnection, mail_to: list[str], *, mail_cc: list[str] | None = None, mail_bcc: list[str] | None = None, subject: str | None = None, attachment_filename: str | None = None, message: str | None = None, ) -> None: """ Emails parsing results as a zip file via an already-authenticated Microsoft Graph mailbox connection (``/users/{mailbox}/sendMail``), saving a copy to Sent Items. Args: results (dict): Parsing results connection (MSGraphConnection): An already-authenticated Microsoft Graph mailbox connection mail_to (list): A list of addresses to mail to mail_cc (list): A list of addresses to CC mail_bcc (list): A list of addresses to BCC subject (str): Overrides the default message subject attachment_filename (str): Override the default attachment filename message (str): Override the default plain text body """ logger.debug("Emailing report via Microsoft Graph") subject, message, attachments = _build_report_email_content( results, subject=subject, attachment_filename=attachment_filename, message=message, ) # Graph derives the From header from the authenticated mailbox and # ignores message_from; it's still passed for API parity with # send_message()'s signature. connection.send_message( message_from=connection.mailbox_name or "", message_to=mail_to, message_cc=mail_cc, message_bcc=mail_bcc, subject=subject, attachments=attachments, plain_message=message, )
# Backward-compatible aliases parse_forensic_report = parse_failure_report parsed_forensic_reports_to_csv_rows = parsed_failure_reports_to_csv_rows parsed_forensic_reports_to_csv = parsed_failure_reports_to_csv