diff --git a/browser/sandbox.py b/browser/sandbox.py index cd47d43b..307ba369 100644 --- a/browser/sandbox.py +++ b/browser/sandbox.py @@ -28,6 +28,7 @@ def interpret_bypass_queue(mailbox, mode, extra_info): # create a string buffer to store stdout user_std_out = StringIO() + user_property_log = StringIO() with sandbox_helpers.override_print(user_std_out) as fakeprint: code = extra_info['code'] message_schemas = MessageSchema.objects.filter(id=extra_info['msg-id']) @@ -46,6 +47,9 @@ def interpret_bypass_queue(mailbox, mode, extra_info): user_environ['new_message'] = new_message mailbox._imap_client.select_folder(message_schema.folder.name) + # clear the property log at the last possible moment + user_property_log.truncate(0) + mailbox._imap_client.user_property_log = user_property_log # execute the user's code if "on_message" in code: exec(code + "\non_message(new_message)", user_environ) @@ -61,6 +65,8 @@ def interpret_bypass_queue(mailbox, mode, extra_info): elif "on_deadline" in code: exec(code + "\non_deadline(new_message)", user_environ) + mailbox._imap_client.user_property_log = None + except Exception: # Get error message for users if occurs # print out error messages for user @@ -69,6 +75,7 @@ def interpret_bypass_queue(mailbox, mode, extra_info): fakeprint(sandbox_helpers.get_error_as_string_for_user()) finally: msg_log["log"] += user_std_out.getvalue() + msg_log["log"] += user_property_log.getvalue() # msg_log["log"] = "%s\n%s" % (user_std_out.getvalue(), msg_log["log"]) res['appended_log'][message_schema.id] = msg_log diff --git a/engine/models/contact.py b/engine/models/contact.py index f27c5ea6..add7443c 100644 --- a/engine/models/contact.py +++ b/engine/models/contact.py @@ -4,6 +4,7 @@ from schema.youps import ContactSchema, MessageSchema # noqa: F401 ignore unused we use it for typing from django.db.models import Q import logging +from engine.models.helpers import CustomProperty logger = logging.getLogger('youps') # type: logging.Logger @@ -32,7 +33,7 @@ def __eq__(self, other): return False - @property + @CustomProperty def email(self): # type: () -> t.AnyStr """Get the email address associated with this contact @@ -42,7 +43,7 @@ def email(self): """ return self._schema.email - @property + @CustomProperty def aliases(self): # type: () -> t.List[t.AnyStr] """Get all the names associated with this contact @@ -52,7 +53,7 @@ def aliases(self): """ return self._schema.aliases.all().values_list('name', flat=True) - @property + @CustomProperty def name(self): # type: () -> t.AnyStr """Get the name associated with this contact @@ -63,7 +64,7 @@ def name(self): # simply returns the most common alias return self._schema.aliases.order_by('-count').first().name - @property + @CustomProperty def organization(self): # type: () -> t.AnyStr """Get the organization of this contact @@ -73,7 +74,7 @@ def organization(self): """ return self._schema.organization - @property + @CustomProperty def geolocation(self): # type: () -> t.AnyStr """Get the location of this contact @@ -83,7 +84,7 @@ def geolocation(self): """ return self._schema.geolocation - @property + @CustomProperty def messages_to(self): # type: () -> t.List[Message] """Get the Messages which are to this contact @@ -94,7 +95,7 @@ def messages_to(self): from engine.models.message import Message return [Message(message_schema, self._imap_client) for message_schema in self._schema.to_messages.all()] - @property + @CustomProperty def messages_from(self): # type: () -> t.List[Message] """Get the Messages which are from this contact @@ -105,7 +106,7 @@ def messages_from(self): from engine.models.message import Message return [Message(message_schema, self._imap_client) for message_schema in self._schema.from_messages.all()] - @property + @CustomProperty def messages_bcc(self): # type: () -> t.List[Message] """Get the Messages which are bcc this contact @@ -116,7 +117,7 @@ def messages_bcc(self): from engine.models.message import Message return [Message(message_schema, self._imap_client) for message_schema in self._schema.bcc_messages.all()] - @property + @CustomProperty def messages_cc(self): # type: () -> t.List[Message] """Get the Messages which are cc this contact diff --git a/engine/models/folder.py b/engine/models/folder.py index 04791cc3..d13fafe1 100644 --- a/engine/models/folder.py +++ b/engine/models/folder.py @@ -3,11 +3,9 @@ import heapq import logging import typing as t # noqa: F401 ignore unused we use it for typing -import re from datetime import datetime from email.header import decode_header -from email.utils import parseaddr, getaddresses -from string import whitespace +from email.utils import getaddresses from itertools import chain import chardet @@ -25,10 +23,11 @@ NewMessageDataScheduled, RemovedFlagsData) from engine.models.message import Message -from engine.utils import normalize_msg_id, folding_ws_regex, encoded_word_string_regex, header_comment_regex +from engine.utils import normalize_msg_id, FOLDING_WS_RE, ENCODED_WORD_STRING_RE, HEADER_COMMENT_RE from schema.youps import ( # noqa: F401 ignore unused we use it for typing BaseMessage, ContactSchema, ContactAlias, FolderSchema, ImapAccount, MessageSchema, ThreadSchema) +from engine.models.helpers import CustomProperty logger = logging.getLogger('youps') # type: logging.Logger @@ -55,7 +54,7 @@ def __eq__(self, other): return other == self.name return False - @property + @CustomProperty def _uid_next(self): # type: () -> int return self._schema.uid_next @@ -66,7 +65,7 @@ def _uid_next(self, value): self._schema.uid_next = value self._schema.save() - @property + @CustomProperty def _uid_validity(self): # type: () -> int return self._schema.uid_validity @@ -77,7 +76,7 @@ def _uid_validity(self, value): self._schema.uid_validity = value self._schema.save() - @property + @CustomProperty def _highest_mod_seq(self): # type: () -> int return self._schema.highest_mod_seq @@ -88,7 +87,7 @@ def _highest_mod_seq(self, value): self._schema.highest_mod_seq = value self._schema.save() - @property + @CustomProperty def name(self): # type: () -> str return self._schema.name @@ -99,7 +98,7 @@ def name(self, value): self._schema.name = value self._schema.save() - @property + @CustomProperty def _last_seen_uid(self): # type: () -> int return self._schema.last_seen_uid @@ -110,7 +109,7 @@ def _last_seen_uid(self, value): self._schema.last_seen_uid = value self._schema.save() - @property + @CustomProperty def _is_selectable(self): # type: () -> bool return self._schema.is_selectable @@ -121,7 +120,7 @@ def _is_selectable(self, value): self._schema.is_selectable = value self._schema.save() - @property + @CustomProperty def _imap_account(self): # type: () -> ImapAccount return self._schema.imap_account @@ -531,11 +530,11 @@ def _parse_email_header(self, header): return None # replace instance of folding white space with nothing - header = folding_ws_regex.sub('', header) + header = FOLDING_WS_RE.sub('', header) for field in header.split('\r\n'): # header can have multiple encoded parts # we can remove this in python3 but its a bug in python2 - parts = chain.from_iterable(decode_header(f) for f in filter(None, encoded_word_string_regex.split(field))) + parts = chain.from_iterable(decode_header(f) for f in filter(None, ENCODED_WORD_STRING_RE.split(field))) combined_parts = u"" for part in parts: text, encoding = part[0], part[1] @@ -567,8 +566,8 @@ def _parse_email_header(self, header): v = fields[k] # nested comments cannot be queried with regex # this hack removes them from the inside out - while header_comment_regex.search(v): - v = header_comment_regex.sub('', v) + while HEADER_COMMENT_RE.search(v): + v = HEADER_COMMENT_RE.sub('', v) fields[k] = v return fields diff --git a/engine/models/helpers/__init__.py b/engine/models/helpers/__init__.py index e69de29b..a11508a5 100644 --- a/engine/models/helpers/__init__.py +++ b/engine/models/helpers/__init__.py @@ -0,0 +1,3 @@ +from custom_property import CustomProperty + +__all__ = ["CustomProperty"] \ No newline at end of file diff --git a/engine/models/helpers/custom_property.py b/engine/models/helpers/custom_property.py new file mode 100644 index 00000000..9a1d83ef --- /dev/null +++ b/engine/models/helpers/custom_property.py @@ -0,0 +1,131 @@ +from __future__ import print_function +import typing as t + +import logging + + +def _get_class_name(obj): + # type: (object) -> str + """Get the class name from an object + + Returns: + str: the name of the class which represents the object + """ + return obj.__class__.__name__ + + +def _get_method_name(prop): + # type: (t.Callable) -> str + """Get the name of a method + + Args: + prop (t.Callable): a method on an object + + Returns: + str: the name which represents the method + """ + return prop.__name__ + + +def _get_logger(): + # type: () -> logging.Logger + return logging.getLogger('youps') + + +def _log_info(obj, info): + # type: (t.AnyStr) -> None + assert obj._imap_client is not None + if hasattr(obj._imap_client, 'user_property_log') and \ + obj._imap_client.user_property_log is not None and \ + hasattr(obj._imap_client, "nested_log") and \ + obj._imap_client.nested_log is False: + _get_logger().critical("HEEEEEEEEEEERE") + user_property_log = obj._imap_client.user_property_log + user_property_log.write(info) + user_property_log.write(u"\n") + + +def _set_in_log(obj, value): + # if hasattr(obj._imap_client, 'user_property_log') and obj._imap_client.user_property_log is not None: + obj._imap_client.nested_log = value + +def _is_nested(obj): + # if hasattr(obj._imap_client, 'user_property_log') and obj._imap_client.user_property_log is not None: + if hasattr(obj._imap_client, 'nested_log'): + return obj._imap_client.nested_log + return False + +class CustomProperty(object): + "Emulate PyProperty_Type() in Objects/descrobject.c" + + def __init__(self, fget=None, fset=None, fdel=None, doc=None): + self.fget = fget + self.fset = fset + self.fdel = fdel + if doc is None and fget is not None: + doc = fget.__doc__ + self.__doc__ = doc + + def __get__(self, obj, objtype=None): + if obj is None: + return self + if self.fget is None: + raise AttributeError("unreadable attribute") + + if _is_nested(obj): + return self.fget(obj) + + _set_in_log(obj, True) + + # value we are getting or wrapping around + value = self.fget(obj) + + # name of the class we are a property on + class_name = _get_class_name(obj) + # name of the property we're wrapping around + property_name = _get_method_name(self.fget) + + info_string = u"get {c}.{p}\t{v}".format( + c=class_name, p=property_name, v=value) + _set_in_log(obj, False) + + if not property_name.startswith("_"): + _log_info(obj, info_string) + + return value + + def __set__(self, obj, new_value): + if self.fset is None: + raise AttributeError("can't set attribute") + if _is_nested(obj): + self.fset(obj, new_value) + return + # TODO doesn't really make sense normally but we want to fget for logging + if self.fget is None: + raise AttributeError("unreadable attribute") + + _set_in_log(obj, True) + curr_value = self.fget(obj) + class_name = _get_class_name(obj) + property_name = _get_method_name(self.fset) + info_string = u"set {c}.{p}\t{v} -> {nv}".format( + c=class_name, p=property_name, v=curr_value, nv=new_value) + self.fset(obj, new_value) + _set_in_log(obj, False) + + if not property_name.startswith("_"): + _log_info(obj, info_string) + + def __delete__(self, obj): + if self.fdel is None: + raise AttributeError("can't delete attribute") + self.fdel(obj) + + def getter(self, fget): + return type(self)(fget, self.fset, self.fdel, self.__doc__) + + def setter(self, fset): + return type(self)(self.fget, fset, self.fdel, self.__doc__) + + def deleter(self, fdel): + return type(self)(self.fget, self.fset, fdel, self.__doc__) diff --git a/engine/models/helpers/message_helpers.py b/engine/models/helpers/message_helpers.py index f824b947..06fb4ff0 100644 --- a/engine/models/helpers/message_helpers.py +++ b/engine/models/helpers/message_helpers.py @@ -2,8 +2,9 @@ import logging import pprint import typing as t -from collections import Sequence +from collections import Sequence, namedtuple from contextlib import contextmanager +from itertools import izip from engine.utils import InvalidFlagException, is_gmail_label @@ -190,4 +191,54 @@ def _save_flags(message, flags): message._schema.flags = flags message._schema.save() - message._flags = flags \ No newline at end of file + message._flags = flags + + +# TODO test this attachment parsing stuff somoe more and make it more legible + +Part = namedtuple('Part', ['maintype', 'subtype', 'parameters', 'id_', + 'description', 'encoding', 'size'] + ) + +def _walk_bodystructure(part): + yield part + if part.is_multipart: + for sub_part in part[0]: + for p in _walk_bodystructure(sub_part): + yield p + +def _pairwise(iterable): + "s -> (s0, s1), (s2, s3), (s4, s5), ..." + a = iter(iterable) + return izip(a, a) + +def _parse_part(part): + # type: (t.Tuple) -> Part + assert not part.is_multipart + part = list(part) + parameter_dict = {k: v for k, v in _pairwise(list(part[2]))} + part[2] = parameter_dict + parsed_part = Part(*(list(part)[:7])) + return parsed_part + +def get_attachments(message): + import pprint + response = message._imap_client.fetch(message._uid, ['BODYSTRUCTURE']) + if message._uid not in response: + raise RuntimeError('Failed to get message content') + response = response[message._uid] + + # get the rfc data we're looking for + if 'BODYSTRUCTURE' not in response: + logger.critical('%s:%s response: %s' % + (message.folder, message, pprint.pformat(response))) + logger.critical("%s did not return BODYSTRUCTURE" % message) + raise RuntimeError("Failed to get message attachment names") + bodystructure = response['BODYSTRUCTURE'] + + parts = [_parse_part(p) for p in _walk_bodystructure(bodystructure) if not p.is_multipart] + file_names = [] + for part in parts: + if 'NAME' in part.parameters: + file_names.append(part.parameters['NAME']) + return file_names diff --git a/engine/models/message.py b/engine/models/message.py index fc5141c5..e4dd0aab 100644 --- a/engine/models/message.py +++ b/engine/models/message.py @@ -27,7 +27,7 @@ ImapAccount, MessageSchema, TaskManager) from smtp_handler.utils import format_email_address, get_attachments from engine.utils import IsNotGmailException -from engine.models.helpers import message_helpers +from engine.models.helpers import message_helpers, CustomProperty userLogger = logging.getLogger('youps.user') # type: logging.Logger logger = logging.getLogger('youps') # type: logging.Logger @@ -100,12 +100,12 @@ def __eq__(self, other): return self._schema == other._schema return False - @property + @CustomProperty def _imap_account(self): # type: () -> ImapAccount return self._schema.imap_account - @property + @CustomProperty def _uid(self): # type: () -> int return self._schema.uid @@ -116,7 +116,7 @@ def _uid(self, value): self._schema.uid = value self._schema.save() - @property + @CustomProperty def _msn(self): # type: () -> int return self._schema.msn @@ -127,12 +127,12 @@ def _msn(self, value): self._schema.msn = value self._schema.save() - @property + @CustomProperty def _message_id(self): # type: () -> int return self._schema.base_message.message_id - @property + @CustomProperty def flags(self): # type: () -> t.List[t.AnyStr] """Get the flags on the message @@ -142,7 +142,7 @@ def flags(self): """ return self._flags if self._is_simulate else self._schema.flags - @property + @CustomProperty def in_reply_to(self): # type: () -> t.List[t.AnyStr] """Get the message ids in the in_reply_to field @@ -152,7 +152,7 @@ def in_reply_to(self): """ return self._schema.base_message.in_reply_to - @property + @CustomProperty def references(self): # type: () -> t.List[t.AnyStr] """Get the message ids in the references field @@ -162,7 +162,7 @@ def references(self): """ return self._schema.base_message.references - @property + @CustomProperty def deadline(self): # type: () -> t.AnyStr """Get the user-defined deadline of the message @@ -184,7 +184,7 @@ def deadline(self, value): self._schema.base_message.deadline = value self._schema.base_message.save() - @property + @CustomProperty def subject(self): # type: () -> t.AnyStr """Get the Subject of the message @@ -194,7 +194,7 @@ def subject(self): """ return self._schema.base_message.subject - @property + @CustomProperty def thread(self): # type: () -> t.Optional[Thread] from engine.models.thread import Thread @@ -203,7 +203,7 @@ def thread(self): # TODO we should create the thread otherwise return None - @property + @CustomProperty def date(self): # type: () -> datetime """Get the date and time that the message was sent @@ -213,7 +213,7 @@ def date(self): """ return self._schema.base_message.date - @property + @CustomProperty def is_read(self): # type: () -> bool """Get if the message has been read @@ -223,7 +223,7 @@ def is_read(self): """ return '\\Seen' in self.flags - @property + @CustomProperty def is_unread(self): # type: () -> bool """Get if the message is unread @@ -233,7 +233,7 @@ def is_unread(self): """ return not self.is_read - @property + @CustomProperty def is_deleted(self): # type: () -> bool """Get if the message has been deleted @@ -243,7 +243,7 @@ def is_deleted(self): """ return '\\Deleted' in self.flags - @property + @CustomProperty def is_recent(self): # type: () -> bool """Get if the message is recent @@ -254,7 +254,7 @@ def is_recent(self): # TODO we will automatically remove the RECENT flag unless we make our imapclient ReadOnly return '\\Recent' in self.flags - @property + @CustomProperty def to(self): # type: () -> t.List[Contact] """Get the Contacts the message is addressed to @@ -265,7 +265,7 @@ def to(self): return [Contact(contact_schema, self._imap_client) for contact_schema in self._schema.base_message.to.all()] - @property + @CustomProperty def from_(self): # type: () -> Contact """Get the Contact the message is addressed from @@ -275,7 +275,7 @@ def from_(self): """ return Contact(self._schema.base_message.from_m, self._imap_client) if self._schema.base_message.from_m else None - @property + @CustomProperty def sender(self): # type: () -> Contact """Get the Contact the message is addressed from @@ -287,7 +287,7 @@ def sender(self): """ return self.from_ - @property + @CustomProperty def reply_to(self): # type: () -> t.List[Contact] """Get the Contacts the message is replied to @@ -300,7 +300,7 @@ def reply_to(self): """ return [Contact(contact_schema, self._imap_client) for contact_schema in self._schema.base_message.reply_to.all()] - @property + @CustomProperty def cc(self): # type: () -> t.List[Contact] """Get the Contacts the message is cced to @@ -310,7 +310,7 @@ def cc(self): """ return [Contact(contact_schema, self._imap_client) for contact_schema in self._schema.base_message.cc.all()] - @property + @CustomProperty def bcc(self): # type: () -> t.List[Contact] """Get the Contacts the message is bcced to @@ -320,7 +320,7 @@ def bcc(self): """ return [Contact(contact_schema, self._imap_client) for contact_schema in self._schema.base_message.bcc.all()] - @property + @CustomProperty def recipients(self): # type: () -> t.List[Contact] """Shortcut method to get a list of all the recipients of an email. @@ -333,7 +333,7 @@ def recipients(self): """ return list(set(chain(self.to, self.cc, self.bcc))) - @property + @CustomProperty def folder(self): # type: () -> Folder """Get the Folder the message is contained in @@ -344,7 +344,7 @@ def folder(self): from engine.models.folder import Folder return Folder(self._schema.folder, self._imap_client) - @property + @CustomProperty def content(self, return_only_text=True): # type: () -> t.AnyStr """Get the content of the message @@ -453,6 +453,10 @@ def contains(self, string): """ return string in self.content + @CustomProperty + def attachments(self): + return message_helpers.get_attachments(self) + def reply(self, to=[], cc=[], bcc=[], content=""): # type: (t.Iterable[t.AnyStr], t.Iterable[t.AnyStr], t.Iterable[t.AnyStr], t.AnyStr) -> None """Reply to the sender of this message diff --git a/engine/models/thread.py b/engine/models/thread.py index c3986b2b..f7b531f8 100644 --- a/engine/models/thread.py +++ b/engine/models/thread.py @@ -1,16 +1,14 @@ from __future__ import division, print_function, unicode_literals +import logging import typing as t # noqa: F401 ignore unused we use it for typing +from itertools import chain, ifilter from imapclient import IMAPClient # noqa: F401 ignore unused we use it for typing -from schema.youps import MessageSchema, ThreadSchema, FolderSchema # noqa: F401 ignore unused we use it for typing - from engine.models.message import Message - -from itertools import chain, ifilter - -import logging +from engine.models.helpers import CustomProperty +from schema.youps import (FolderSchema, MessageSchema, ThreadSchema) # noqa: F401 ignore unused we use it for typing logger = logging.getLogger('youps') # type: logging.Logger @@ -48,7 +46,7 @@ def __eq__(self, other): def __len__(self): return self._schema.baseMessages.all().count() - @property + @CustomProperty def messages(self): # type: () -> t.List[Message] """Get the messages associated with the thread diff --git a/engine/utils.py b/engine/utils.py index 690f2460..8cf75251 100644 --- a/engine/utils.py +++ b/engine/utils.py @@ -107,7 +107,7 @@ def normalize_msg_id(message_id): str: standard message_id for comparison with other message ids """ # clean up empty strings - message_ids = message_id_split_regex.findall(message_id) + message_ids = MESSAGE_ID_SPLIT_RE.findall(message_id) # TODO better sanity checking from rfc5322 # sanity check @@ -261,18 +261,20 @@ def sortkey(node): # REGEXES # Match carriage return new lines followed by whitespace. For example "\r\n \t\t" # this is used to indicate multi line fields in email headers -folding_ws_regex = re.compile(r'\r\n[\ \t]+') +FOLDING_WS_RE = re.compile(r'\r\n[\ \t]+') # Match encoded-word strings in the form =?charset?q?Hello_World?= # this is used to indicate encoding in email headers # the regex used in python2 decode_header is incorrect, this is from the # python3 version and can be removed when/if the project moves to python3 -encoded_word_string_regex = re.compile(r'(=\?[^?]*?\?[qQbB]\?.*?\?=)', re.VERBOSE | re.MULTILINE) +ENCODED_WORD_STRING_RE = re.compile(r'(=\?[^?]*?\?[qQbB]\?.*?\?=)', re.VERBOSE | re.MULTILINE) # basically find Parentheses containing text surrounded by optional white space # TODO might have to rewrite with RFC5322 -header_comment_regex = re.compile(r'\((?:(?:[\ \t]*\r\n){0,1}[\ \t]*[\x01-\x08\x0B\x0C\x0E-\x1F\x21-\x27\x2A-\x5B\x5D-\x7F]|\\[\x01-\x09\x0B\x0C\x0E-\x7F])*(?:[\ \t]*\r\n){0,1}[\ \t]*\)') +HEADER_COMMENT_RE = re.compile(r'\((?:(?:[\ \t]*\r\n){0,1}[\ \t]*[\x01-\x08\x0B\x0C\x0E-\x1F\x21-\x27\x2A-\x5B\x5D-\x7F]|\\[\x01-\x09\x0B\x0C\x0E-\x7F])*(?:[\ \t]*\r\n){0,1}[\ \t]*\)') -message_id_split_regex = re.compile(r'<(.*?)>') \ No newline at end of file +MESSAGE_ID_SPLIT_RE = re.compile(r'<(.*?)>') + +ATTACHMENTS_RE = re.compile(r'(?<="name"\s").+?(?=")') diff --git a/schema/youps.py b/schema/youps.py index c07cc795..72503e7c 100644 --- a/schema/youps.py +++ b/schema/youps.py @@ -67,6 +67,8 @@ class FolderSchema(models.Model): last_seen_uid = models.IntegerField(default=-1) # the highest mod seq useful for limiting getting the flags highest_mod_seq = models.IntegerField(default=-1) + # whether or not the folder is selectable + is_selectable = models.BooleanField(default=False) class Meta: db_table = "youps_folder"