parsing/decrypting of complex PGP messages in Kotlin

[tomholub]

part of #1051 but first need to do #1060

from https://github.com/FlowCrypt/flowcrypt-mobile-core/blob/master/source/mobile-interface/endpoints.ts

Unit tests to copy: everything that starts with parseDecryptMsg at https://github.com/FlowCrypt/flowcrypt-mobile-core/blob/master/source/test.ts

current implementation:

 public parseDecryptMsg = async (uncheckedReq: any, data: Buffers): Promise<Buffers> => {
 const { keys: kisWithPp, msgPwd, isEmail } = ValidateInput.parseDecryptMsg(uncheckedReq);
 const rawBlocks: MsgBlock[] = []; // contains parsed, unprocessed / possibly encrypted data
 let rawSigned: string | undefined = undefined;
 let subject: string | undefined = undefined;
 if (isEmail) {
 const { blocks, rawSignedContent, headers } = await Mime.process(Buf.concat(data));
 subject = String(headers['subject']);
 rawSigned = rawSignedContent;
 rawBlocks.push(...blocks);
 } else {
 rawBlocks.push(MsgBlock.fromContent('encryptedMsg', new Buf(Buf.concat(data))));
 }
 const sequentialProcessedBlocks: MsgBlock[] = []; // contains decrypted or otherwise formatted data
 for (const rawBlock of rawBlocks) {
 if ((rawBlock.type === 'signedMsg' || rawBlock.type === 'signedHtml') && rawBlock.signature) {
 const verify = await PgpMsg.verifyDetached({ sigText: Buf.fromUtfStr(rawBlock.signature), plaintext: Buf.with(rawSigned || rawBlock.content) });
 if (rawBlock.type === 'signedHtml') {
 sequentialProcessedBlocks.push({ type: 'verifiedMsg', content: Xss.htmlSanitizeKeepBasicTags(rawBlock.content.toString()), verifyRes: verify, complete: true });
 } else { // text
 sequentialProcessedBlocks.push({ type: 'verifiedMsg', content: Str.asEscapedHtml(rawBlock.content.toString()), verifyRes: verify, complete: true });
 }
 } else if (rawBlock.type === 'encryptedMsg' || rawBlock.type === 'signedMsg') {
 const decryptRes = await PgpMsg.decrypt({ kisWithPp, msgPwd, encryptedData: Buf.with(rawBlock.content) });
 if (decryptRes.success) {
 if (decryptRes.isEncrypted) {
 const formatted = await MsgBlockParser.fmtDecryptedAsSanitizedHtmlBlocks(decryptRes.content);
 sequentialProcessedBlocks.push(...formatted.blocks);
 subject = formatted.subject || subject;
 } else {
 // treating as text, converting to html - what about plain signed html? This could produce html tags
 // although hopefully, that would, typically, result in the `(rawBlock.type === 'signedMsg' || rawBlock.type === 'signedHtml')` block above
 // the only time I can imagine it screwing up down here is if it was a signed-only message that was actually fully armored (text not visible) with a mime msg inside
 // ... -> in which case the user would I think see full mime content?
 sequentialProcessedBlocks.push({ type: 'verifiedMsg', content: Str.asEscapedHtml(decryptRes.content.toUtfStr()), complete: true, verifyRes: decryptRes.signature });
 }
 } else {
 decryptRes.message = undefined;
 sequentialProcessedBlocks.push({
 type: 'decryptErr',
 content: decryptRes.error.type === DecryptErrTypes.noMdc ? decryptRes.content! : rawBlock.content,
 decryptErr: decryptRes,
 complete: true
 });
 }
 } else if (rawBlock.type === 'encryptedAtt' && rawBlock.attMeta && /^(0x)?[A-Fa-f0-9]{16,40}\.asc\.pgp$/.test(rawBlock.attMeta.name || '')) {
 // encrypted pubkey attached
 const decryptRes = await PgpMsg.decrypt({ kisWithPp, msgPwd, encryptedData: Buf.with(rawBlock.attMeta.data || '') });
 if (decryptRes.content) {
 sequentialProcessedBlocks.push({ type: 'publicKey', content: decryptRes.content.toString(), complete: true });
 } else {
 sequentialProcessedBlocks.push(rawBlock); // will show as encryptedAtt
 }
 } else {
 sequentialProcessedBlocks.push(rawBlock);
 }
 }
 const msgContentBlocks: MsgBlock[] = [];
 const blocks: MsgBlock[] = [];
 let replyType = 'plain';
 for (const block of sequentialProcessedBlocks) { // fix/adjust/format blocks before returning it over JSON
 if (block.content instanceof Buf) { // cannot JSON-serialize Buf
 block.content = isContentBlock(block.type) ? block.content.toUtfStr() : block.content.toRawBytesStr();
 } else if (block.attMeta && block.attMeta.data instanceof Uint8Array) {
 // converting to base64-encoded string instead of uint8 for JSON serilization
 // value actually replaced to a string, but type remains Uint8Array type set to satisfy TS
 // no longer used below, only gets passed to be serialized as JSON - later consumed by iOS or Android app
 block.attMeta.data = Buf.fromUint8(block.attMeta.data).toBase64Str() as any as Uint8Array;
 }
 if (block.type === 'decryptedHtml' || block.type === 'decryptedText' || block.type === 'decryptedAtt') {
 replyType = 'encrypted';
 }
 if (block.type === 'publicKey') {
 if (!block.keyDetails) { // this could eventually be moved into detectBlocks, which would make it async
 const { keys } = await PgpKey.normalize(block.content);
 if (keys.length) {
 for (const pub of keys) {
 blocks.push({ type: 'publicKey', content: pub.armor(), complete: true, keyDetails: await PgpKey.details(pub) });
 }
 } else {
 blocks.push({
 type: 'decryptErr',
 content: block.content,
 complete: true,
 decryptErr: {
 success: false,
 error: { type: 'format' as DecryptErrTypes, message: 'Badly formatted public key' },
 longids: { message: [], matching: [], chosen: [], needPassphrase: [] }
 }
 });
 }
 } else {
 blocks.push(block);
 }
 } else if (isContentBlock(block.type)) {
 msgContentBlocks.push(block);
 } else if (Mime.isPlainImgAtt(block)) {
 msgContentBlocks.push(block);
 } else if (block.type !== 'plainAtt') {
 blocks.push(block);
 }
 }
 const { contentBlock, text } = fmtContentBlock(msgContentBlocks);
 blocks.unshift(contentBlock);
 // data represent one JSON-stringified block per line. This is so that it can be read as a stream later
 return fmtRes({ text, replyType, subject }, Buf.fromUtfStr(blocks.map(b => JSON.stringify(b)).join('\n')));
 }

[tomholub]

@IvanPizhenko this will be the next step. Here you take a MIME message and split it into chunks for further processing. You will use your recently added decrypt method here.

Inputs: keys, data, isEmail
outputs: text, replyType, subject, blocks: MsgBlock[]

But first a few questions to @DenBond7 about usage: on Android do you use both isEmail:true and isEmail:false or just one of them?

[tomholub]

You can skip the signature verification parts completely for now: therefore instead of verifiedMsg you can return signedMsg without trying to verify it.

[tomholub]

@DenBond7 also what do you currently use for MIME parsing and how? In the code above we use Mime.process which itself is not a simple method.

I will split this issue into smaller chunks and list them here, else it would be hard to code and review in whole.