User Tools

Site Tools


docs:tips_n_tricks:shellskripting:code:mbox.html

Scripts for decomposing e-mail folders in mbox format

Find corrupted mails causing dovecot replication to fail

Assuming you have moved all mail of respective IMAP folder to some local bzip-ed mbox folder

Check you free inode count1) before and while doing this!
# Install //reformail//:
apt-get install maildrop
~/scripts/unpack_mbox.bz2.sh *.bz2
 
# In case you have archived one broken IMAP folder into
# multiple bzip-ed mbox files, you might want to merge
# their resulting directories here, e.g.
#
# mkdir "${imapname}.d" && for dir in "${imapname}-"* ; do mv -iv "$dir"/*/ "${imapname}.d/" </dev/null ; done
# rmdir -v "${imapname}-"*/
# # take care of double Message-IDs
# for dir in "${imapname}-"*/*/ ; do mv -iv "$dir"/* "${imapname}.d/${dir#*/}" </dev/null ; done
 
# for mboxdir in <list of newly created dirs>
cd "$mboxdir"
# Remove "duplicate" mails, that differ only in Status: or X-Keywords: header lines:
for dir in */ ; do ( cd "$dir" ; \ls -1 mail.* | ( read first ; while read file ; do diff -s -I '^\(X-UID: [0-9]\+\|Status: *[RO]\?\|X-Keywords: *\)$' "$first" "$file" && \rm -v "$file" ; done ) ) ; done
# Sort Message-ID folders for count of non-identical mails:
for dir in */ ; do count="`ls -1 \"$dir\" | wc -l`" ; printf "%4d %s\n" "$count" "$dir" ; mkdir -p "$count" ; mv -iv "$dir" "$count" ; done

Files

unpack_mbox.bz2.sh
#! /bin/sh
 
set -o errexit
 
for mboxbz2 in "$@"
do
  workdir="`mktemp -d \"${mboxbz2%.bz2}.XXXXXX\"`"
  export workdir
  bunzip2 -c "$mboxbz2" \
  | reformail -s sh -c 'cat > `mktemp "$workdir"/mail.XXXXXX`; echo -n .'  
  for file in "${workdir}"/mail.*
  do
    # you need a 'head -1' hier as mails might contain other mails - or
    # mbox source might have been corrupted
    msgid="`sed -ne 's/^Message-ID: <\(.*\)>$/\1/pg' \"$file\"|head -1`"
    mkdir -p "${workdir}/${msgid}"
    mv -iv "$file" "${workdir}/${msgid}" </dev/null
  done
done
merge_dirs_and_cope_with_multiple_msgid_lines.sh
#!/bin/sh
workdir="$1".d \
&& mkdir -p "$workdir" \
&& for file in "${workdir%.d}"-?.*/*/mail.*
do
  msgid="`sed -ne 's/^Message-ID: <\(.*\)>$/\1/pg' \"$file\"|head -1`"
  mkdir -p "${workdir}/${msgid}"
  mv -iv "$file" "${workdir}/${msgid}" </dev/null
done
rmdir -p "${workdir%.d}-"?.*/*/\n
rmdir -p "${workdir%.d}-"?.*/
1)
df -i
docs/tips_n_tricks/shellskripting/code/mbox.html.txt · Last modified: 01.02.2019 12:45 UTC by peter