First implementation

This commit is contained in:
Daniel Demus
2020-01-19 19:38:16 +01:00
parent d5ed1eadf8
commit 3969d987bd
10 changed files with 406 additions and 35 deletions
+33 -1
View File
@@ -1,2 +1,34 @@
# sa-train
A bash script and systemd setup that run sa-learn on mailbox folders
A bash script and systemd setup that run sa-learn on all mailbox folders for users that
are members of the satrain group.
It is assumed that read mails in a Spam folder in the root of a user's maildir has spam,
while everthing else is ham. A directory called SpamSuspect is assumed to contain mail
that needs to be manually categorised by being moved into Spam or another folder.
All the specific names can be configured in either */etc/sa-train.conf* or
*~/.sa-train/config* where the user configuration overrides the system configuration.
**SETTINGS:**
Possible configuration values are (with the default value if not otherwise specified
in config):
**spam_folder .Spam**
The subfolder in the user's mailbox folder that contains mail to train as spam.
Only read mails are used, so if the MTA delivers mail marked as spam by spamassassin
to this folder as unread, the user can verify them by marking them as read, after
which sa-train will add them to the bayesian filter.
**max_age 90**
The maximum age in days of mails to train spamassassin with. This is probably only
relevant the first time training is performed. Normally sa-train only examines mails
that have a modified date more recent than the newest mail in the last training
session. To re-train spamassassin remove the ~/.sa-train/last* files, delete the
bayesian database and run sa-train.sh as root.
**spammed_group sa-train**
The name of the unix group, the members of which will have their mailboxes used to
train the spamassassin bayesian filter. Each users filter is trained using their
specific mailboxes, so the filtering conform to their patterns of spam.
*This setting is only relevant in the global config file and is ignored in the user configs.*
-10
View File
@@ -1,10 +0,0 @@
#!/bin/bash
# Learn spam from Spam Maildir folder, ham from INBOX
/usr/bin/sa-learn --spam --no-sync ~/Maildir/.Spam/cur
/usr/bin/sa-learn --ham --no-sync ~/Maildir/cur
/usr/bin/sa-learn --sync
# Removes all files from ~/Maildir/.Spam/cur that are older than
# 31 days ago
find ~/Maildir/.Spam/cur -mtime +30 -exec rm -f {} \;
+2
View File
@@ -0,0 +1,2 @@
# The default group of spamassassin trainers
g sa-train - -
+28 -23
View File
@@ -1,53 +1,58 @@
#!/bin/bash
shopt -s extglob
while getopts u:d:p:f: option
do
case "${option}"
in
u) USER=${OPTARG};;
d) DATE=${OPTARG};;
p) PRODUCT=${OPTARG};;
f) FORMAT=${OPTARG};;
esac
done
if [ -r /etc/sa-train.conf ]; then
while read -r name value; do
if [[ -n "$name" && ! "$name" =~ "$\w*#" ]]; then
typeset "$name=$value"
fi
done < /etc/sa-train.conf
fi
if [ -r ~/.sa-train/config ]; then
while read -r name value; do
if [[ -n "$name" && ! "$name" =~ "$\w*#" ]]; then
typeset "$name=$value"
fi
done < ~/.sa-train/config
fi
local SPAMFOLDER=${spam_folder:-.Spam}
typeset -i "MAX_AGE=${max_age:-90}"
function examine
{
local subdir=$1
if [[ -n "$subdir" ]]; then
if [[ -n "$subdir" -a "$subdir" == "${subdir%/}" ]]; then
subdir="$subdir/"
fi
if [[ ! -d ./${subdir}cur ]]; then
if [[ ! -d ./${subdir}cur ]]; then
return
fi
local action=ham
if [[ ! -z "$2" ]]; then
action=$2
fi
local action=${2:-ham}
echo Learning from $subdir as $action
pushd ${subdir}cur
if [[ -f ~/.sa-learn/last$1 ]]; then
find -H . -type f -regex ".*,[^,]*S[^,]*$" -newer ~/.sa-learn/last$1 -exec sa-learn --$action --no-sync {} \+;
if [[ -f ~/.sa-train/last$1 ]]; then
find -H . -type f -regex ".*,[^,]*S[^,]*$" -newer ~/.sa-train/last$1 -exec sa-learn --$action --no-sync {} \+;
else
find . -type f -regex ".,[^,]*S[^,]*$" -mtime -30 -exec sa-learn --$action --no-sync {} \+;
find . -type f -regex ".,[^,]*S[^,]*$" -mtime -$MAX_AGE -exec sa-learn --$action --no-sync {} \+;
fi
rm -f ~/.sa-learn/last$1
rm -f ~/.sa-train/last$1
if [[ $( ls *,*([!,])S*([!,]) 2>/dev/null ) ]]; then
ln -s $( realpath $( ls -1t *,*([!,])S*([!,]) | head -1) ) ~/.sa-learn/last$1;
ln -s $( realpath $( ls -1t *,*([!,])S*([!,]) | head -1) ) ~/.sa-train/last$1;
fi
popd
}
if [ -d ~/Maildir ]; then
mkdir -p ~/.sa-learn
mkdir -p ~/.sa-train
pushd ~/Maildir
examine .Spam spam
examine .$SPAMFOLDER spam
examine
examine .Root
for inboxdir in ./.INBOX*; do
+12
View File
@@ -0,0 +1,12 @@
# An example config file for sa-train. Copy it to
# ~/.sa-learn/config and uncomment the relevant lines
# to create user specific configuration
# Unix group that will be trained
# spammed_group sa-train
# The user mailbox spam subfolder
# spam_folder .Spam
# The maximum age in days of mails to train spamassassin with
# max_age 90
+134
View File
@@ -0,0 +1,134 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="FlexmarkProjectSettings">
<FlexmarkHtmlSettings flexmarkSpecExampleRendering="0" flexmarkSpecExampleRenderHtml="false">
<flexmarkSectionLanguages>
<option name="1" value="Markdown" />
<option name="2" value="HTML" />
<option name="3" value="flexmark-ast:1" />
</flexmarkSectionLanguages>
</FlexmarkHtmlSettings>
</component>
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
</profile>
<version value="1.0" />
</component>
<component name="MarkdownEnhProjectSettings">
<AnnotatorSettings targetHasSpaces="true" linkCaseMismatch="true" wikiCaseMismatch="true" wikiLinkHasDashes="true" notUnderWikiHome="true" targetNotWikiPageExt="true" notUnderSourceWikiHome="true" targetNameHasAnchor="true" targetPathHasAnchor="true" wikiLinkHasSlash="true" wikiLinkHasSubdir="true" wikiLinkHasOnlyAnchor="true" linkTargetsWikiHasExt="true" linkTargetsWikiHasBadExt="true" notUnderSameRepo="true" targetNotUnderVcs="false" linkNeedsExt="true" linkHasBadExt="true" linkTargetNeedsExt="true" linkTargetHasBadExt="true" wikiLinkNotInWiki="true" imageTargetNotInRaw="true" repoRelativeAcrossVcsRoots="true" multipleWikiTargetsMatch="true" unresolvedLinkReference="true" linkIsIgnored="true" anchorIsIgnored="true" anchorIsUnresolved="true" anchorLineReferenceIsUnresolved="true" anchorLineReferenceFormat="true" anchorHasDuplicates="true" abbreviationDuplicates="true" abbreviationNotUsed="true" attributeIdDuplicateDefinition="true" attributeIdNotUsed="true" footnoteDuplicateDefinition="true" footnoteUnresolved="true" footnoteDuplicates="true" footnoteNotUsed="true" macroDuplicateDefinition="true" macroUnresolved="true" macroDuplicates="true" macroNotUsed="true" referenceDuplicateDefinition="true" referenceUnresolved="true" referenceDuplicates="true" referenceNotUsed="true" referenceUnresolvedNumericId="true" enumRefDuplicateDefinition="true" enumRefUnresolved="true" enumRefDuplicates="true" enumRefNotUsed="true" enumRefLinkUnresolved="true" enumRefLinkDuplicates="true" simTocUpdateNeeded="true" simTocTitleSpaceNeeded="true" />
<HtmlExportSettings updateOnSave="false" parentDir="" targetDir="" cssDir="css" scriptDir="js" plainHtml="false" imageDir="" copyLinkedImages="false" imagePathType="0" targetPathType="2" targetExt="" useTargetExt="false" noCssNoScripts="false" useElementStyleAttribute="false" linkToExportedHtml="true" exportOnSettingsChange="true" regenerateOnProjectOpen="false" linkFormatType="HTTP_ABSOLUTE" />
<LinkMapSettings>
<textMaps />
</LinkMapSettings>
</component>
<component name="MarkdownNavigatorHistory">
<PasteImageHistory checkeredTransparentBackground="false" filename="image" directory="" onPasteImageTargetRef="3" onPasteLinkText="0" onPasteImageElement="1" onPasteLinkElement="1" onPasteReferenceElement="2" cornerRadius="20" borderColor="0" transparentColor="16777215" borderWidth="1" trimTop="0" trimBottom="0" trimLeft="0" trimRight="0" transparent="false" roundCorners="false" showPreview="true" bordered="false" scaled="false" cropped="false" hideInapplicableOperations="false" preserveLinkFormat="false" scale="50" scalingInterpolation="1" transparentTolerance="0" saveAsDefaultOnOK="false" linkFormat="0" addHighlights="false" showHighlightCoordinates="true" showHighlights="false" mouseSelectionAddsHighlight="false" outerFilled="false" outerFillColor="0" outerFillTransparent="true" outerFillAlpha="30">
<highlightList />
<directories />
<filenames />
</PasteImageHistory>
<CopyImageHistory checkeredTransparentBackground="false" filename="image" directory="" onPasteImageTargetRef="3" onPasteLinkText="0" onPasteImageElement="1" onPasteLinkElement="1" onPasteReferenceElement="2" cornerRadius="20" borderColor="0" transparentColor="16777215" borderWidth="1" trimTop="0" trimBottom="0" trimLeft="0" trimRight="0" transparent="false" roundCorners="false" showPreview="true" bordered="false" scaled="false" cropped="false" hideInapplicableOperations="false" preserveLinkFormat="false" scale="50" scalingInterpolation="1" transparentTolerance="0" saveAsDefaultOnOK="false" linkFormat="0" addHighlights="false" showHighlightCoordinates="true" showHighlights="false" mouseSelectionAddsHighlight="false" outerFilled="false" outerFillColor="0" outerFillTransparent="true" outerFillAlpha="30">
<highlightList />
<directories />
<filenames />
</CopyImageHistory>
<PasteLinkHistory onPasteImageTargetRef="3" onPasteTargetRef="1" onPasteLinkText="0" onPasteImageElement="1" onPasteLinkElement="1" onPasteWikiElement="2" onPasteReferenceElement="2" hideInapplicableOperations="false" preserveLinkFormat="false" useHeadingForLinkText="false" linkFormat="0" saveAsDefaultOnOK="false" />
<TableToJsonHistory>
<entries />
</TableToJsonHistory>
<TableSortHistory>
<entries />
</TableSortHistory>
</component>
<component name="MarkdownProjectSettings">
<PreviewSettings splitEditorLayout="SPLIT" splitEditorPreview="PREVIEW" useGrayscaleRendering="false" zoomFactor="1.5" maxImageWidth="0" synchronizePreviewPosition="true" highlightPreviewType="NONE" highlightFadeOut="5" highlightOnTyping="true" synchronizeSourcePosition="true" verticallyAlignSourceAndPreviewSyncPosition="true" showSearchHighlightsInPreview="false" showSelectionInPreview="true" lastLayoutSetsDefault="false">
<PanelProvider>
<provider providerId="com.vladsch.idea.multimarkdown.editor.swing.html.panel" providerName="Default - Swing" />
</PanelProvider>
</PreviewSettings>
<ParserSettings gitHubSyntaxChange="false" correctedInvalidSettings="false" emojiShortcuts="1" emojiImages="0">
<PegdownExtensions>
<option name="ANCHORLINKS" value="true" />
<option name="ATXHEADERSPACE" value="true" />
<option name="AUTOLINKS" value="true" />
<option name="FENCED_CODE_BLOCKS" value="true" />
<option name="INTELLIJ_DUMMY_IDENTIFIER" value="true" />
<option name="RELAXEDHRULES" value="true" />
<option name="STRIKETHROUGH" value="true" />
<option name="TABLES" value="true" />
<option name="TASKLISTITEMS" value="true" />
<option name="WIKILINKS" value="true" />
</PegdownExtensions>
<ParserOptions>
<option name="COMMONMARK_LISTS" value="true" />
<option name="EMOJI_SHORTCUTS" value="true" />
<option name="GFM_TABLE_RENDERING" value="true" />
<option name="GITHUB_WIKI_LINKS" value="true" />
<option name="PRODUCTION_SPEC_PARSER" value="true" />
<option name="SIM_TOC_BLANK_LINE_SPACER" value="true" />
</ParserOptions>
</ParserSettings>
<HtmlSettings headerTopEnabled="false" headerBottomEnabled="false" bodyTopEnabled="false" bodyBottomEnabled="false" addPageHeader="true" imageUriSerials="false" addDocTypeHtml="true" noParaTags="false" plantUmlConversion="0">
<GeneratorProvider>
<provider providerId="com.vladsch.idea.multimarkdown.editor.swing.html.generator" providerName="Default Swing HTML Generator" />
</GeneratorProvider>
<headerTop />
<headerBottom />
<bodyTop />
<bodyBottom />
</HtmlSettings>
<CssSettings previewScheme="UI_SCHEME" cssUri="" isCssUriEnabled="false" isCssUriSerial="true" isCssTextEnabled="false" isDynamicPageWidth="true">
<StylesheetProvider>
<provider providerId="com.vladsch.idea.multimarkdown.editor.swing.html.css" providerName="Default Swing Stylesheet" />
</StylesheetProvider>
<ScriptProviders />
<cssText />
<cssUriHistory />
</CssSettings>
</component>
<component name="ProjectInspectionProfilesVisibleTreeState">
<entry key="Project Default">
<profile-state>
<expanded-state>
<State />
<State>
<id>CodeNarc - Ruleset rulesets/imports.xml</id>
</State>
<State>
<id>CodeNarc - Ruleset rulesets/unused.xml</id>
</State>
<State>
<id>Data flowGroovy</id>
</State>
<State>
<id>Declaration redundancyGroovy</id>
</State>
<State>
<id>Error handlingGroovy</id>
</State>
<State>
<id>GSPGrailsGroovy</id>
</State>
<State>
<id>GrailsGroovy</id>
</State>
<State>
<id>Groovy</id>
</State>
</expanded-state>
<selected-state>
<State>
<id>YAML</id>
</State>
</selected-state>
</profile-state>
</entry>
</component>
<component name="ProjectRootManager">
<output url="file://$PROJECT_DIR$/out" />
</component>
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
+92
View File
@@ -0,0 +1,92 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="0a6d7486-f55c-4a52-904a-439d06f3a6c4" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/sa-train.spec" afterDir="false" />
<change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/sa-bayes-learn.sh" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/sa-train-user.sh" beforeDir="false" afterPath="$PROJECT_DIR$/sa-train-user.sh" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="DefaultGradleProjectSettings">
<option name="isMigrated" value="true" />
</component>
<component name="FavoritesManager">
<favorites_list name="CodeIris Favorites" />
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="MacroExpansionManager">
<option name="directoryName" value="BbcYWgga" />
</component>
<component name="ProjectId" id="1WabYRDJQgW4A9lUzNZnnrtr3Ni" />
<component name="ProjectViewState">
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showExcludedFiles" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent">
<property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/../scratch" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="project.structure.last.edited" value="Modules" />
<property name="project.structure.proportion" value="0.0" />
<property name="project.structure.side.proportion" value="0.0" />
<property name="settings.editor.selected.configurable" value="Errors" />
</component>
<component name="RunManager">
<configuration default="true" type="#org.jetbrains.idea.devkit.run.PluginConfigurationType">
<module name="" />
<option name="VM_PARAMETERS" value="-Xmx512m -Xms256m -ea" />
<option name="PROGRAM_PARAMETERS" value="" />
<alternative-path path="IDEA jdk" alternative-path-enabled="true" />
<predefined_log_file enabled="true" id="idea.log" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>
<component name="ServiceViewManager">
<option name="viewStates">
<list>
<serviceView>
<treeState>
<expand />
<select />
</treeState>
</serviceView>
</list>
</option>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="0a6d7486-f55c-4a52-904a-439d06f3a6c4" name="Default Changelist" comment="" />
<created>1579390699450</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1579390699450</updated>
<workItem from="1579390703726" duration="736000" />
</task>
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="WindowStateProjectService">
<state x="1415" y="676" width="1000" height="837" key="#Project_Structure" timestamp="1579390727933">
<screen x="0" y="32" width="3840" height="2128" />
</state>
<state x="1415" y="676" width="1000" height="837" key="#Project_Structure/0.32.3840.2128@0.32.3840.2128" timestamp="1579390727933" />
</component>
</project>
+1 -1
View File
@@ -1,5 +1,5 @@
[Unit]
DescriptionExecutes sa-learn for each folder in each member of a given group Maildir folder
Description=Executes sa-learn for each folder in each member of a given group Maildir folder
Wants=sa-train.timer
[Service]
Executable
+26
View File
@@ -0,0 +1,26 @@
#!/bin/bash
if [ "$(id -u)" != "0" ]; then
echo "This script must be run as root"
exit 1
fi
if [ -r /etc/sa-train.conf ]; then
while read -r name value; do
if [[ ! "$name" =~ "$\w*#" ]]; then
typeset "$name=$value"
fi
done < /etc/sa-train.conf
fi
spammed_group=${spammed_group:-sa-train}
SPAMMED_USERS=$(groupmems -l -g "$spammed_group" 2> /dev/null)
if [ -z "$SPAMMED_USERS" ]; then
echo "No users have been added to the \"$spammed_group\" group. Canceling execution."
exit 2
fi
for USERNAME in "$SPAMMED_USERS"; do
sudo -U $USERNAME -H /usr/bin/sa-train-user.sh
done
+78
View File
@@ -0,0 +1,78 @@
Name: sa-train
Version: 1.0.0
Release: 1%{?dist}
Summary: Run sa-learn on all mailbox folders for users that are members of the certain group.
License: GPL-v3
URL: https://github.com/danieldemus/sa-train
Source0: https://github.com/danieldemus/sa-train/archive/master.zip
BuildRequires:
Requires:
%description
A bash script and systemd setup that run sa-learn on all mailbox folders for users that
are members of the satrain group.
It is assumed that read mails in a Spam folder in the root of a user's maildir has spam,
while everthing else is ham. A directory called SpamSuspect is assumed to contain mail
that needs to be manually categorised by being moved into Spam or another folder.
All the specific names can be configured.
%prep
%setup -c sa-learn-%{Version}
%build
%install
rm -rf %{buildroot}
mkdir -p %{buildroot}%{_bindir} \
%{buildroot}%{_sysconfdir}/ \
%{buildroot}%{_unitdir} \
%{buildroot}%{_pkgdocdir}
install sa-train.sh %{buildroot}%{_bindir}/sa-train.sh
install sa-train-user.sh %{buildroot}%{_bindir}/sa-train-user.sh
#README
install -m644 README.md %{buildroot}%{_pkgdocdir}
#Config file
install sa-train.conf %{buildroot}%{_sysconfdir}/sa-train.conf
# Systemd
install -m644 sa-train.service %{buildroot}%{_unitdir}/%{name}.service
install -m644 sa-train.timer %{buildroot}%{_unitdir}/%{name}.timer
install -m644 sa-train.target %{buildroot}%{_unitdir}/%{name}.target
#install systemd sysuser file
install -Dpm 644 sa-train-user.conf %{buildroot}%{_sysusersdir}/%{name}.conf
%post
%systemd_post sa-learn.timer
%preun
%systemd_preun sa-learn.timer
%postun
%systemd_postun_with_restart sa-learn.timer
%files
%doc %{_pkgdocdir}/README.md
%defattr(0644,root,root,0755)
%config(noreplace) %{_sysconfdir}/sa-train.conf
%defattr(-,root,root,-)
%{_sysusersdir}/sa-train.conf
%{_bindir}/sa-train.sh
%{_bindir}/sa-train-user.sh
%{_unitdir}/sa-train.service
%{_unitdir}/sa-train.target
%{_unitdir}/sa-train.timer
%changelog
* Sat Jan 18 2020 Daniel Demus <dde@nine.dk>
- Initial release