[mail] Add mail content processing (#14345)

* [mail] Add mail content processing

Signed-off-by: Jan N. Klug <github@klug.nrw>
This commit is contained in:
J-N-K 2023-07-03 09:07:27 +02:00 committed by GitHub
parent 99e78d84b5
commit f0bdeff81b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 226 additions and 60 deletions

View File

@ -185,7 +185,7 @@
/bundles/org.openhab.binding.luxom/ @jesperskriasoft
/bundles/org.openhab.binding.luxtronikheatpump/ @sgiehl
/bundles/org.openhab.binding.magentatv/ @markus7017
/bundles/org.openhab.binding.mail/ @openhab/add-ons-maintainers
/bundles/org.openhab.binding.mail/ @J-N-K
/bundles/org.openhab.binding.max/ @marcelrv
/bundles/org.openhab.binding.mcd/ @simon-dengler
/bundles/org.openhab.binding.mcp23017/ @aogorek

View File

@ -40,16 +40,42 @@ Default ports are `143` (for `PLAIN` and `STARTTLS`) and `993` (for `SSL`) in th
## Channels
There are no channels for the `smtp` thing.
The `imap` and `pop3` things can be extended with `mailcount`-type channels.
The `imap` and `pop3` things can be extended with `mailcount`- and `content`-type channels.
### Type `mailcount`
Each channel has two parameters: `folder` and `type`.
The `folder` is mandatory and denotes the folder name on the given account.
You can either use the root folder like (e.g. "INBOX") or a subdirectory of your structure (e.g. "INBOX.Sent" or "INBOX.Junk").
The `type` parameter can be `UNREAD` or `TOTAL` (default).
Channels with type `UNREAD` give the number on unread mails in that folder.
### Type `content`
The `content` type channel presents the contents of an unread mail.
If the message is a MIME- or MIME-multipart message, all parts are concatenated.
The content is converted to a plain string without processing (i.e. HTML tags are still present).
In most cases the mail content needs further processing in rules to trigger appropriate action.
Each channel has five parameters: `folder`, `subject`, `sender`, `transformation` and `markAsRead`.
The `folder` is mandatory and denotes the folder name on the given account.
You can either use the root folder like (e.g. "INBOX") or a subdirectory of your structure (e.g. "INBOX.Sent" or "INBOX.Junk").
`subject` and `sender` can be used to filter the messages that are processed by the channel.
Filters use regular expressions (e.g. `.*DHL.*` as `sender` would match all From-addresses that contain "DHL").
If a parameter is left empty, no filter is applied.
The `transformation` is applied before setting the channel status.
Transformations can be chained by separating them with the mathematical intersection character "∩", e.g. `REGEX:.*Shipment-Status: ([a-z]+).*∩MAP:status.map` would first extract a character string with a regular expression and then apply the given MAP transformation on the result.
Please note that the values will be discarded if one transformation fails (e.g. REGEX did not match).
This means that you can also use it to filter certain emails e.g. `REGEX:(.*Sendungsbenachrichtigung.*)` would only match for mails containing the string "Sendungsbenachrichtigung" but output the whole message.
Since with each refresh all unread mails are processed the same message content would be sent to the channel multiple times.
This can be prevented by setting `markAsRead` to `true` (default is `false`), which marks all processed messages as read.
## Full Example
mail.things:
@ -61,6 +87,7 @@ Thing mail:imap:sampleimap [ hostname="imap.example.com", security="SSL", userna
Channels:
Type mailcount : inbox_total [ folder="INBOX", type="TOTAL" ]
Type mailcount : inbox_unread [ folder="INBOX", type="UNREAD" ]
Type content : fedex_notification [ folder="INBOX" sender="Fedex.*" markAsRead="true" ]
}
```
@ -69,6 +96,7 @@ mail.items:
```java
Number InboxTotal "INBOX [%d]" { channel="mail:imap:sampleimap:inbox_total" }
Number InboxUnread "INBOX Unread [%d]" { channel="mail:imap:sampleimap:inbox_unread" }
String FedexNotification { channel="mail:imap:sampleimap:fedex_notification" }
```
mail.sitemap:

View File

@ -38,4 +38,5 @@ public class MailBindingConstants {
Arrays.asList(THING_TYPE_SMTPSERVER, THING_TYPE_IMAPSERVER, THING_TYPE_POP3SERVER));
public static final ChannelTypeUID CHANNEL_TYPE_UID_FOLDER_MAILCOUNT = new ChannelTypeUID(BINDING_ID, "mailcount");
public static final ChannelTypeUID CHANNEL_TYPE_UID_MAIL_CONTENT = new ChannelTypeUID(BINDING_ID, "content");
}

View File

@ -12,12 +12,14 @@
*/
package org.openhab.binding.mail.internal;
import org.eclipse.jdt.annotation.NonNullByDefault;
/**
* The {@link MailCountChannelType} enum for folder mail count type
*
* @author Jan N. Klug - Initial contribution
*/
@NonNullByDefault
public enum MailCountChannelType {
UNREAD,
TOTAL

View File

@ -29,8 +29,8 @@ import org.osgi.service.component.annotations.Component;
*
* @author Jan N. Klug - Initial contribution
*/
@NonNullByDefault
@Component(configurationPid = "binding.mail", service = ThingHandlerFactory.class)
@NonNullByDefault
public class MailHandlerFactory extends BaseThingHandlerFactory {
@Override

View File

@ -13,29 +13,41 @@
package org.openhab.binding.mail.internal;
import static org.openhab.binding.mail.internal.MailBindingConstants.CHANNEL_TYPE_UID_FOLDER_MAILCOUNT;
import static org.openhab.binding.mail.internal.MailBindingConstants.CHANNEL_TYPE_UID_MAIL_CONTENT;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Properties;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.mail.Address;
import javax.mail.Flags;
import javax.mail.Folder;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.Session;
import javax.mail.Store;
import javax.mail.internet.MimeMessage;
import javax.mail.internet.MimeMultipart;
import javax.mail.search.FlagTerm;
import org.eclipse.jdt.annotation.NonNullByDefault;
import org.eclipse.jdt.annotation.Nullable;
import org.openhab.binding.mail.internal.config.POP3IMAPChannelConfig;
import org.openhab.binding.mail.internal.config.POP3IMAPConfig;
import org.openhab.binding.mail.internal.config.POP3IMAPContentChannelConfig;
import org.openhab.binding.mail.internal.config.POP3IMAPMailCountChannelConfig;
import org.openhab.core.library.types.DecimalType;
import org.openhab.core.library.types.StringType;
import org.openhab.core.thing.Channel;
import org.openhab.core.thing.ChannelUID;
import org.openhab.core.thing.Thing;
import org.openhab.core.thing.ThingStatus;
import org.openhab.core.thing.ThingStatusDetail;
import org.openhab.core.thing.binding.BaseThingHandler;
import org.openhab.core.thing.binding.generic.ChannelTransformation;
import org.openhab.core.types.Command;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -76,23 +88,16 @@ public class POP3IMAPHandler extends BaseThingHandler {
if (config.port == 0) {
switch (protocol) {
case "imap":
config.port = 143;
break;
case "imaps":
config.port = 993;
break;
case "pop3":
config.port = 110;
break;
case "pop3s":
config.port = 995;
break;
default:
case "imap" -> config.port = 143;
case "imaps" -> config.port = 993;
case "pop3" -> config.port = 110;
case "pop3s" -> config.port = 995;
default -> {
updateStatus(ThingStatus.OFFLINE, ThingStatusDetail.CONFIGURATION_ERROR);
return;
}
}
}
refreshTask = scheduler.scheduleWithFixedDelay(this::refresh, 0, config.refresh, TimeUnit.SECONDS);
updateStatus(ThingStatus.ONLINE);
@ -109,6 +114,9 @@ public class POP3IMAPHandler extends BaseThingHandler {
}
private void refresh() {
if (Thread.currentThread().isInterrupted()) {
return;
}
Properties props = new Properties();
props.setProperty("mail." + baseProtocol + ".starttls.enable", "true");
props.setProperty("mail.store.protocol", protocol);
@ -119,8 +127,8 @@ public class POP3IMAPHandler extends BaseThingHandler {
for (Channel channel : thing.getChannels()) {
if (CHANNEL_TYPE_UID_FOLDER_MAILCOUNT.equals(channel.getChannelTypeUID())) {
final POP3IMAPChannelConfig channelConfig = channel.getConfiguration()
.as(POP3IMAPChannelConfig.class);
final POP3IMAPMailCountChannelConfig channelConfig = channel.getConfiguration()
.as(POP3IMAPMailCountChannelConfig.class);
final String folderName = channelConfig.folder;
if (folderName == null || folderName.isEmpty()) {
logger.info("missing or empty folder name in channel {}", channel.getUID());
@ -133,14 +141,65 @@ public class POP3IMAPHandler extends BaseThingHandler {
updateState(channel.getUID(), new DecimalType(
mailbox.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false)).length));
}
} catch (MessagingException e) {
throw e;
}
}
} else if (CHANNEL_TYPE_UID_MAIL_CONTENT.equals(channel.getChannelTypeUID())) {
final POP3IMAPContentChannelConfig channelConfig = channel.getConfiguration()
.as(POP3IMAPContentChannelConfig.class);
final String folderName = channelConfig.folder;
if (folderName == null || folderName.isEmpty()) {
logger.info("missing or empty folder name in channel '{}'", channel.getUID());
} else {
try (Folder mailbox = store.getFolder(folderName)) {
mailbox.open(channelConfig.markAsRead ? Folder.READ_WRITE : Folder.READ_ONLY);
Message[] messages = mailbox.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false));
for (Message message : messages) {
String subject = message.getSubject();
Address[] senders = message.getFrom();
String sender = senders == null ? ""
: Stream.of(senders).map(Address::toString).collect(Collectors.joining(","));
logger.debug("Processing `{}` from `{}`", subject, sender);
if (!channelConfig.subject.isBlank() && !subject.matches(channelConfig.subject)) {
logger.trace("Subject '{}' did not pass subject filter", subject);
continue;
}
if (!channelConfig.sender.isBlank() && !sender.matches(channelConfig.sender)) {
logger.trace("Sender '{}' did not pass filter '{}'", subject, channelConfig.sender);
continue;
}
Object rawContent = message.getContent();
String contentAsString;
if (rawContent instanceof String) {
logger.trace("Detected plain text message");
contentAsString = (String) rawContent;
} else if (rawContent instanceof MimeMessage mimeMessage) {
logger.trace("Detected MIME message");
try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
mimeMessage.writeTo(os);
contentAsString = os.toString();
}
} else if (rawContent instanceof MimeMultipart mimeMultipart) {
logger.trace("Detected MIME multipart message");
try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
mimeMultipart.writeTo(os);
contentAsString = os.toString();
}
} else {
logger.warn(
"Failed to convert mail content from '{}' with subject '{}', to String: {}",
sender, subject, rawContent.getClass());
continue;
}
logger.trace("Found content '{}'", contentAsString);
new ChannelTransformation(channelConfig.transformation).apply(contentAsString)
.ifPresent(result -> updateState(channel.getUID(), new StringType(result)));
}
}
}
}
} catch (MessagingException e) {
logger.info("error when trying to refresh IMAP: {}", e.getMessage());
}
} catch (MessagingException | IOException e) {
logger.info("Failed refreshing IMAP for thing '{}': {}", thing.getUID(), e.getMessage());
}
}
}

View File

@ -12,12 +12,14 @@
*/
package org.openhab.binding.mail.internal;
import org.eclipse.jdt.annotation.NonNullByDefault;
/**
* The {@link ServerSecurity} enum contains security configuration options
*
* @author Jan N. Klug - Initial contribution
*/
@NonNullByDefault
public enum ServerSecurity {
PLAIN,
SSL,

View File

@ -21,7 +21,6 @@ import org.openhab.binding.mail.internal.ServerSecurity;
*
* @author Jan N. Klug - Initial contribution
*/
@NonNullByDefault
public class BaseConfig {
public @Nullable String hostname;

View File

@ -19,7 +19,6 @@ import org.eclipse.jdt.annotation.NonNullByDefault;
*
* @author Jan N. Klug - Initial contribution
*/
@NonNullByDefault
public class POP3IMAPConfig extends BaseConfig {
public int refresh = 60;

View File

@ -0,0 +1,31 @@
/**
* Copyright (c) 2010-2023 Contributors to the openHAB project
*
* See the NOTICE file(s) distributed with this work for additional
* information.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.openhab.binding.mail.internal.config;
import org.eclipse.jdt.annotation.NonNullByDefault;
import org.eclipse.jdt.annotation.Nullable;
/**
* The {@link POP3IMAPContentChannelConfig} class contains fields mapping thing configuration parameters.
*
* @author Jan N. Klug - Initial contribution
*/
@NonNullByDefault
public class POP3IMAPContentChannelConfig {
public @Nullable String folder;
public String subject = "";
public String sender = "";
public @Nullable String transformation;
public boolean markAsRead = false;
}

View File

@ -17,13 +17,12 @@ import org.eclipse.jdt.annotation.Nullable;
import org.openhab.binding.mail.internal.MailCountChannelType;
/**
* The {@link POP3IMAPChannelConfig} class contains fields mapping thing configuration parameters.
* The {@link POP3IMAPMailCountChannelConfig} class contains fields mapping thing configuration parameters.
*
* @author Jan N. Klug - Initial contribution
*/
@NonNullByDefault
public class POP3IMAPChannelConfig {
public class POP3IMAPMailCountChannelConfig {
public @Nullable String folder;
public MailCountChannelType type = MailCountChannelType.TOTAL;
}

View File

@ -20,7 +20,6 @@ import org.eclipse.jdt.annotation.Nullable;
*
* @author Jan N. Klug - Initial contribution
*/
@NonNullByDefault
public class SMTPConfig extends BaseConfig {
public @Nullable String sender;

View File

@ -20,6 +20,31 @@ thing-type.config.mail.smtp.port.description = Default values are 25 for plain/S
thing-type.config.mail.smtp.sender.label = Sender
thing-type.config.mail.smtp.sender.description = Default sender address for mail
# channel types
channel-type.mail.content.label = Content
channel-type.mail.content.description = Mail content as String (with subject filter and content transformation).
channel-type.mail.mailcount.label = Mail Count
channel-type.mail.mailcount.description = Number of emails in folder
# channel types config
channel-type.config.mail.content.folder.label = Folder Name
channel-type.config.mail.content.markAsRead.label = Mark As Read
channel-type.config.mail.content.markAsRead.description = Mark a processed mail as read and prevent further processing.
channel-type.config.mail.content.sender.label = Sender Filter
channel-type.config.mail.content.sender.description = A (regular expression) filter for the mail sender address.
channel-type.config.mail.content.subject.label = Subject Filter
channel-type.config.mail.content.subject.description = A (regular expression) filter for the mail subject.
channel-type.config.mail.content.transformation.label = Transformation
channel-type.config.mail.content.transformation.description = Transformation pattern used when processing messages. Multiple transformation can be chained using "∩".
channel-type.config.mail.mailcount.folder.label = Folder Name
channel-type.config.mail.mailcount.type.label = Counter Type
channel-type.config.mail.mailcount.type.option.UNREAD = Unread
channel-type.config.mail.mailcount.type.option.TOTAL = Total
# thing types config
config.hostname.label = Server Hostname
config.password.label = SMTP Server Password
config.port.label = Server Port
@ -31,18 +56,6 @@ config.security.option.STARTTLS = STARTTLS
config.security.option.SSL = SSL/TLS
config.username.label = SMTP Server Username
# channel types
channel-type.mail.mailcount.label = Mail Count
channel-type.mail.mailcount.description = Number of emails in folder
# channel types config
channel-type.config.mail.mailcount.folder.label = Folder Name
channel-type.config.mail.mailcount.type.label = Counter Type
channel-type.config.mail.mailcount.type.option.UNREAD = Unread
channel-type.config.mail.mailcount.type.option.TOTAL = Total
# actions
addHeaderActionLabel = add a mail header

View File

@ -15,12 +15,12 @@
<parameter name="hostname" type="text" required="true">
<label>@text/config.hostname.label</label>
</parameter>
<parameter name="port" type="text" required="false">
<parameter name="port" type="text">
<label>@text/config.port.label</label>
<description>Default values are 25 for plain/STARTTLS and 465 for SSL/TLS</description>
<advanced>true</advanced>
</parameter>
<parameter name="security" type="text" required="false">
<parameter name="security" type="text">
<label>@text/config.security.label</label>
<options>
<option value="PLAIN">@text/config.security.option.PLAIN</option>
@ -30,28 +30,28 @@
<limitToOptions>true</limitToOptions>
<default>PLAIN</default>
</parameter>
<parameter name="username" type="text" required="false">
<parameter name="username" type="text">
<label>@text/config.username.label</label>
</parameter>
<parameter name="password" type="text" required="false">
<parameter name="password" type="text">
<label>@text/config.password.label</label>
<context>password</context>
</parameter>
</config-description>
</thing-type>
<thing-type id="imap" extensible="mailcount">
<thing-type id="imap" extensible="mailcount,content">
<label>IMAP Server</label>
<description>Used for receiving emails</description>
<config-description>
<parameter name="hostname" type="text" required="true">
<label>@text/config.hostname.label</label>
</parameter>
<parameter name="port" type="text" required="false">
<parameter name="port" type="text">
<label>@text/config.port.label</label>
<description>Default values are 143 for plain/STARTTLS and 993 for SSL/TLS</description>
<advanced>true</advanced>
</parameter>
<parameter name="security" type="text" required="false">
<parameter name="security" type="text">
<label>@text/config.security.label</label>
<options>
<option value="PLAIN">@text/config.security.option.PLAIN</option>
@ -68,26 +68,26 @@
<label>@text/config.password.label</label>
<context>password</context>
</parameter>
<parameter name="refresh" type="integer" required="false">
<parameter name="refresh" type="integer">
<label>@text/config.refresh.label</label>
<description>@text/config.refresh.description</description>
<default>60</default>
</parameter>
</config-description>
</thing-type>
<thing-type id="pop3" extensible="mailcount">
<thing-type id="pop3" extensible="mailcount,content">
<label>POP3 Server</label>
<description>Used for receiving emails</description>
<config-description>
<parameter name="hostname" type="text" required="true">
<label>@text/config.hostname.label</label>
</parameter>
<parameter name="port" type="text" required="false">
<parameter name="port" type="text">
<label>@text/config.port.label</label>
<description>Default values are 110 for plain/STARTTLS and 995 for SSL/TLS</description>
<advanced>true</advanced>
</parameter>
<parameter name="security" type="text" required="false">
<parameter name="security" type="text">
<label>@text/config.security.label</label>
<options>
<option value="PLAIN">@text/config.security.option.PLAIN</option>
@ -104,7 +104,7 @@
<label>@text/config.password.label</label>
<context>password</context>
</parameter>
<parameter name="refresh" type="integer" required="false">
<parameter name="refresh" type="integer">
<label>@text/config.refresh.label</label>
<description>@text/config.refresh.description</description>
<default>60</default>
@ -121,7 +121,7 @@
<parameter name="folder" type="text" required="true">
<label>Folder Name</label>
</parameter>
<parameter name="type" type="text" required="false">
<parameter name="type" type="text">
<label>Counter Type</label>
<options>
<option value="UNREAD">Unread</option>
@ -132,4 +132,33 @@
</parameter>
</config-description>
</channel-type>
<channel-type id="content">
<item-type>String</item-type>
<label>Content</label>
<description>Mail content as String (with subject filter and content transformation).</description>
<state readOnly="true"/>
<config-description>
<parameter name="folder" type="text" required="true">
<label>Folder Name</label>
</parameter>
<parameter name="subject" type="text">
<label>Subject Filter</label>
<description>A (regular expression) filter for the mail subject.</description>
</parameter>
<parameter name="sender" type="text">
<label>Sender Filter</label>
<description>A (regular expression) filter for the mail sender address.</description>
</parameter>
<parameter name="transformation" type="text">
<label>Transformation</label>
<description>Transformation pattern used when processing messages. Multiple transformation can be chained using "∩".</description>
</parameter>
<parameter name="markAsRead" type="boolean">
<label>Mark As Read</label>
<description>Mark a processed mail as read and prevent further processing.</description>
<default>false</default>
</parameter>
</config-description>
</channel-type>
</thing:thing-descriptions>

View File

@ -10,7 +10,7 @@
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.openhab.binding.mail;
package org.openhab.binding.mail.internal;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.MatcherAssert.assertThat;
@ -29,15 +29,15 @@ import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.HtmlEmail;
import org.apache.commons.mail.MultiPartEmail;
import org.apache.commons.mail.SimpleEmail;
import org.eclipse.jdt.annotation.NonNullByDefault;
import org.junit.jupiter.api.Test;
import org.openhab.binding.mail.internal.MailBuilder;
/**
* The {@link MailBuilderTest} class defines tests for the {@link MailBuilder} class
*
* @author Jan N. Klug - Initial contribution
*/
@NonNullByDefault
public class MailBuilderTest {
private static final String TEST_STRING = "test";

View File

@ -22,4 +22,9 @@
<suppress files=".+org.openhab.binding.yeelight.+" checks="OutsideOfLibExternalLibrariesCheck" />
<!-- suppress header checks for imported and patched apache commons-io files in logreader binding -->
<suppress files=".+org.openhab.binding.logreader.internal.thirdparty.commonsio.+" checks="ParameterizedRegexpHeaderCheck|AuthorTagCheck" />
<!-- Mail: Do not check org.apache.commons.mail usage -->
<suppress files=".+[\\/]mail[\\/].+[\\/]MailBuilder\.java" checks="ForbiddenPackageUsageCheck"/>
<suppress files=".+[\\/]mail[\\/].+[\\/]MailBuilderTest\.java" checks="ForbiddenPackageUsageCheck"/>
<suppress files=".+[\\/]mail[\\/].+[\\/]SMTPHandler\.java" checks="ForbiddenPackageUsageCheck"/>
<suppress files=".+[\\/]mail[\\/].+[\\/]SendMailActions\.java" checks="ForbiddenPackageUsageCheck"/>
</suppressions>