FetchBodyParts: decode source in the workers

Previously the workers returned a mixture of decoded / encoded parts.
This lead to a whole bunch of issues.
This commit changes the msgviewer and the commands to assume parts to already
be decoded

8 files changed, 29 insertions, 117 deletions

diff --git a/commands/msg/forward.go b/commands/msg/forward.go
index 7570177..35d276e 100644
--- a/commands/msg/forward.go
+++ b/commands/msg/forward.go
@@ -10,9 +10,6 @@ import (
 	"path"
 	"strings"
 
-	"github.com/emersion/go-message"
-	"github.com/emersion/go-message/mail"
-
 	"git.sr.ht/~sircmpwn/aerc/models"
 	"git.sr.ht/~sircmpwn/aerc/widgets"
 	"git.sr.ht/~sircmpwn/getopt"
@@ -138,28 +135,9 @@ func (forward) Execute(aerc *widgets.Aerc, args []string) error {
 
 		// TODO: something more intelligent than fetching the 1st part
 		// TODO: add attachments!
-		store.FetchBodyPart(msg.Uid, []int{1}, func(reader io.Reader) {
-			header := message.Header{}
-			header.SetText(
-				"Content-Transfer-Encoding", msg.BodyStructure.Encoding)
-			header.SetContentType(
-				msg.BodyStructure.MIMEType, msg.BodyStructure.Params)
-			header.SetText("Content-Description", msg.BodyStructure.Description)
-			entity, err := message.New(header, reader)
-			if err != nil {
-				// TODO: Do something with the error
-				addTab()
-				return
-			}
-			mreader := mail.NewReader(entity)
-			part, err := mreader.NextPart()
-			if err != nil {
-				// TODO: Do something with the error
-				addTab()
-				return
-			}
+		store.FetchBodyPart(msg.Uid, msg.BodyStructure, []int{1}, func(reader io.Reader) {
 			buf := new(bytes.Buffer)
-			buf.ReadFrom(part.Body)
+			buf.ReadFrom(reader)
 			defaults["Original"] = buf.String()
 			addTab()
 		})
diff --git a/commands/msg/pipe.go b/commands/msg/pipe.go
index 2faa5de..001577c 100644
--- a/commands/msg/pipe.go
+++ b/commands/msg/pipe.go
@@ -1,13 +1,10 @@
 package msg
 
 import (
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
-	"mime/quotedprintable"
 	"os/exec"
-	"strings"
 	"time"
 
 	"git.sr.ht/~sircmpwn/aerc/commands"
@@ -129,14 +126,7 @@ func (Pipe) Execute(aerc *widgets.Aerc, args []string) error {
 		})
 	} else if pipePart {
 		p := provider.SelectedMessagePart()
-		p.Store.FetchBodyPart(p.Msg.Uid, p.Index, func(reader io.Reader) {
-			// email parts are encoded as 7bit (plaintext), quoted-printable, or base64
-			if strings.EqualFold(p.Part.Encoding, "base64") {
-				reader = base64.NewDecoder(base64.StdEncoding, reader)
-			} else if strings.EqualFold(p.Part.Encoding, "quoted-printable") {
-				reader = quotedprintable.NewReader(reader)
-			}
-
+		p.Store.FetchBodyPart(p.Msg.Uid, p.Msg.BodyStructure, p.Index, func(reader io.Reader) {
 			if background {
 				doExec(reader)
 			} else {
diff --git a/commands/msg/reply.go b/commands/msg/reply.go
index 359c5dd..a7379d7 100644
--- a/commands/msg/reply.go
+++ b/commands/msg/reply.go
@@ -9,9 +9,6 @@ import (
 	"strings"
 
 	"git.sr.ht/~sircmpwn/getopt"
-	"github.com/emersion/go-message"
-	_ "github.com/emersion/go-message/charset"
-	"github.com/emersion/go-message/mail"
 
 	"git.sr.ht/~sircmpwn/aerc/models"
 	"git.sr.ht/~sircmpwn/aerc/widgets"
@@ -155,56 +152,9 @@ func (reply) Execute(aerc *widgets.Aerc, args []string) error {
 			template = aerc.Config().Templates.QuotedReply
 		}
 
-		store.FetchBodyPart(msg.Uid, []int{1}, func(reader io.Reader) {
-			header := message.Header{}
-			if len(msg.BodyStructure.Parts) > 0 {
-				partID := 0 // TODO: will we always choose first msg part?
-				header.SetText(
-					"Content-Transfer-Encoding", msg.BodyStructure.Parts[partID].Encoding)
-				if msg.BodyStructure.Parts[partID].MIMESubType == "" {
-					header.SetContentType(
-						msg.BodyStructure.Parts[partID].MIMEType,
-						msg.BodyStructure.Parts[partID].Params)
-				} else {
-					// include SubType if defined (text/plain, text/html, ...)
-					header.SetContentType(
-						fmt.Sprintf("%s/%s", msg.BodyStructure.Parts[partID].MIMEType,
-							msg.BodyStructure.Parts[partID].MIMESubType),
-						msg.BodyStructure.Parts[partID].Params)
-				}
-				header.SetText("Content-Description", msg.BodyStructure.Parts[partID].Description)
-			} else { // Parts has no headers, so we use global headers info
-				header.SetText(
-					"Content-Transfer-Encoding", msg.BodyStructure.Encoding)
-				if msg.BodyStructure.MIMESubType == "" {
-					header.SetContentType(
-						msg.BodyStructure.MIMEType,
-						msg.BodyStructure.Params)
-				} else {
-					// include SubType if defined (text/plain, text/html, ...)
-					header.SetContentType(
-						fmt.Sprintf("%s/%s", msg.BodyStructure.MIMEType,
-							msg.BodyStructure.MIMESubType),
-						msg.BodyStructure.Params)
-				}
-				header.SetText("Content-Description", msg.BodyStructure.Description)
-			}
-			entity, err := message.New(header, reader)
-			if err != nil {
-				// TODO: Do something with the error
-				addTab()
-				return
-			}
-			mreader := mail.NewReader(entity)
-			part, err := mreader.NextPart()
-			if err != nil {
-				// TODO: Do something with the error
-				addTab()
-				return
-			}
-
+		store.FetchBodyPart(msg.Uid, msg.BodyStructure, []int{1}, func(reader io.Reader) {
 			buf := new(bytes.Buffer)
-			buf.ReadFrom(part.Body)
+			buf.ReadFrom(reader)
 			defaults["Original"] = buf.String()
 			addTab()
 		})
@@ -214,6 +164,7 @@ func (reply) Execute(aerc *widgets.Aerc, args []string) error {
 	}
 }
 
+//TODO (RPB): unused function
 func findPlaintext(bs *models.BodyStructure,
 	path []int) (*models.BodyStructure, []int) {
 
diff --git a/commands/msgview/open.go b/commands/msgview/open.go
index ab023a1..6001d28 100644
--- a/commands/msgview/open.go
+++ b/commands/msgview/open.go
@@ -36,7 +36,7 @@ func (Open) Execute(aerc *widgets.Aerc, args []string) error {
 	mv := aerc.SelectedTab().(*widgets.MessageViewer)
 	p := mv.SelectedMessagePart()
 
-	p.Store.FetchBodyPart(p.Msg.Uid, p.Index, func(reader io.Reader) {
+	p.Store.FetchBodyPart(p.Msg.Uid, p.Msg.BodyStructure, p.Index, func(reader io.Reader) {
 		// email parts are encoded as 7bit (plaintext), quoted-printable, or base64
 
 		if strings.EqualFold(p.Part.Encoding, "base64") {
diff --git a/commands/msgview/save.go b/commands/msgview/save.go
index 99abe0e..c017e70 100644
--- a/commands/msgview/save.go
+++ b/commands/msgview/save.go
@@ -60,7 +60,7 @@ func (Save) Execute(aerc *widgets.Aerc, args []string) error {
 	mv := aerc.SelectedTab().(*widgets.MessageViewer)
 	p := mv.SelectedMessagePart()
 
-	p.Store.FetchBodyPart(p.Msg.Uid, p.Index, func(reader io.Reader) {
+	p.Store.FetchBodyPart(p.Msg.Uid, p.Msg.BodyStructure, p.Index, func(reader io.Reader) {
 		// email parts are encoded as 7bit (plaintext), quoted-printable, or base64
 
 		if strings.EqualFold(p.Part.Encoding, "base64") {
diff --git a/lib/msgstore.go b/lib/msgstore.go
index f67c49f..7209316 100644
--- a/lib/msgstore.go
+++ b/lib/msgstore.go
@@ -127,11 +127,22 @@ func (store *MessageStore) FetchFull(uids []uint32, cb func(io.Reader)) {
 }
 
 func (store *MessageStore) FetchBodyPart(
-	uid uint32, part []int, cb func(io.Reader)) {
+	uid uint32, parent *models.BodyStructure, part []int, cb func(io.Reader)) {
+	partbs, err := parent.PartAtIndex(part)
+	if err != nil {
+		store.worker.Logger.Printf("FetchBodyPart: %v\n", err)
+	}
+	var charset string
+	var ok bool
+	if charset, ok = partbs.Params["charset"]; !ok {
+		charset = ""
+	}
 
 	store.worker.PostAction(&types.FetchMessageBodyPart{
-		Uid:  uid,
-		Part: part,
+		Uid:      uid,
+		Part:     part,
+		Encoding: partbs.Encoding,
+		Charset:  charset,
 	}, func(resp types.WorkerMessage) {
 		msg, ok := resp.(*types.MessageBodyPart)
 		if !ok {
diff --git a/widgets/msgviewer.go b/widgets/msgviewer.go
index cc883fc..32368ef 100644
--- a/widgets/msgviewer.go
+++ b/widgets/msgviewer.go
@@ -10,9 +10,6 @@ import (
 	"strings"
 
 	"github.com/danwakefield/fnmatch"
-	message "github.com/emersion/go-message"
-	_ "github.com/emersion/go-message/charset"
-	"github.com/emersion/go-message/mail"
 	"github.com/gdamore/tcell"
 	"github.com/google/shlex"
 	"github.com/mattn/go-runewidth"
@@ -549,10 +546,6 @@ func (pv *PartViewer) SetSource(reader io.Reader) {
 
 func (pv *PartViewer) attemptCopy() {
 	if pv.source != nil && pv.pager != nil && pv.pager.Process != nil {
-		header := message.Header{}
-		header.SetText("Content-Transfer-Encoding", pv.part.Encoding)
-		header.SetContentType(fmt.Sprintf("%s/%s", pv.part.MIMEType, pv.part.MIMESubType), pv.part.Params)
-		header.SetText("Content-Description", pv.part.Description)
 		if pv.filter != nil {
 			stdout, _ := pv.filter.StdoutPipe()
 			stderr, _ := pv.filter.StderrPipe()
@@ -608,28 +601,15 @@ func (pv *PartViewer) attemptCopy() {
 				pv.pagerin.Write([]byte{'\n'})
 			}
 
-			entity, err := message.New(header, pv.source)
-			if err != nil {
-				pv.err = err
-				pv.Invalidate()
-				return
-			}
-			reader := mail.NewReader(entity)
-			part, err := reader.NextPart()
-			if err != nil {
-				pv.err = err
-				pv.Invalidate()
-				return
-			}
 			if pv.part.MIMEType == "text" {
-				scanner := bufio.NewScanner(part.Body)
+				scanner := bufio.NewScanner(pv.source)
 				for scanner.Scan() {
 					text := scanner.Text()
 					text = ansi.ReplaceAllString(text, "")
 					io.WriteString(pv.sink, text+"\n")
 				}
 			} else {
-				io.Copy(pv.sink, part.Body)
+				io.Copy(pv.sink, pv.source)
 			}
 			pv.sink.Close()
 		}()
@@ -653,7 +633,7 @@ func (pv *PartViewer) Draw(ctx *ui.Context) {
 		return
 	}
 	if !pv.fetched {
-		pv.store.FetchBodyPart(pv.msg.Uid, pv.index, pv.SetSource)
+		pv.store.FetchBodyPart(pv.msg.Uid, pv.msg.BodyStructure, pv.index, pv.SetSource)
 		pv.fetched = true
 	}
 	if pv.err != nil {
diff --git a/worker/types/messages.go b/worker/types/messages.go
index a38ff94..c7d5077 100644
--- a/worker/types/messages.go
+++ b/worker/types/messages.go
@@ -104,8 +104,10 @@ type FetchFullMessages struct {
 
 type FetchMessageBodyPart struct {
 	Message
-	Uid  uint32
-	Part []int
+	Uid      uint32
+	Part     []int
+	Encoding string
+	Charset  string
 }
 
 type DeleteMessages struct {