Introducing emails and pubkeys stores instead of contacts

extension/chrome/dev/ci_unit_test.ts

@@ -13,6 +13,7 @@ import { Wkd } from '../../js/common/api/key-server/wkd.js';
 import { MsgUtil } from '../../js/common/core/crypto/pgp/msg-util.js';
 import { Sks } from '../../js/common/api/key-server/sks.js';
 import { Ui } from '../../js/common/browser/ui.js';
+import { ContactStore } from '../../js/common/platform/store/contact-store.js';
 
 /**
  * importing all libs that are tested in ci tests
@@ -29,7 +30,8 @@ const libs: any[] = [
  Wkd,
  Sks,
  MsgUtil,
- Ui
+ Ui,
+ ContactStore
 ];
 
 // add them to global scope so ci can use them

extension/chrome/elements/add_pubkey.ts

@@ -87,7 +87,7 @@ View.run(class AddPubkeyView extends View {
  try {
  const keyImportUi = new KeyImportUi({ checkEncryption: true });
  const normalized = await keyImportUi.checkPub(String($('.pubkey').val()));
- await ContactStore.save(undefined, await ContactStore.obj({ email: String($('select.email').val()), client: 'pgp', pubkey: normalized }));
+ await ContactStore.save(undefined, await ContactStore.obj({ email: String($('select.email').val()), pubkey: normalized }));
  this.closeDialog();
  } catch (e) {
  if (e instanceof UserAlert) {

extension/chrome/elements/compose-modules/compose-recipients-module.ts

@@ -739,15 +739,8 @@ export class ComposeRecipientsModule extends ViewModule<ComposeView> {
  const contact = await ContactStore.obj({ email: input.email, name: input.name });
  const [storedContact] = await ContactStore.get(undefined, [contact.email]);
  if (storedContact) {
- const toUpdate: ContactUpdate = {};
  if (!storedContact.name && contact.name) {
- toUpdate.name = contact.name;
- }
- if (storedContact.searchable.length !== contact.searchable.length && storedContact.searchable.join(',') !== contact.searchable.join(',')) {
- toUpdate.searchable = contact.searchable;
- }
- if (Object.keys(toUpdate).length) {
- await ContactStore.update(undefined, contact.email, toUpdate);
+ await ContactStore.update(undefined, contact.email, { name: contact.name } as ContactUpdate);
  }
  } else if (!this.failedLookupEmails.includes(contact.email)) {
  toLookup.push(contact);

extension/chrome/elements/compose-modules/compose-render-module.ts

@@ -313,7 +313,6 @@ export class ComposeRenderModule extends ViewModule<ComposeView> {
  await ContactStore.save(undefined, await ContactStore.obj({
  email: key.emails[0],
  name: Str.parseEmail(key.identities[0]).name,
- client: 'pgp',
  pubkey: normalizedPub,
  lastCheck: Date.now(),
  }));

extension/chrome/elements/compose-modules/compose-storage-module.ts

@@ -110,14 +110,7 @@ export class ComposeStorageModule extends ViewModule<ComposeView> {
  lookupResult.pubkey = null; // tslint:disable-line:no-null-keyword
  }
  }
- const client = lookupResult.pgpClient === 'flowcrypt' ? 'cryptup' : 'pgp'; // todo - clean up as "flowcrypt|pgp-other'. Already in storage, fixing involves migration
- const ksContact = await ContactStore.obj({
- email,
- name,
- pubkey: lookupResult.pubkey,
- client: lookupResult.pubkey ? client : undefined,
- lastCheck: Date.now(),
- });
+ const ksContact = await ContactStore.obj({ email, name, pubkey: lookupResult.pubkey, lastCheck: Date.now() });
  if (ksContact.pubkey) {
  this.ksLookupsByEmail[email] = ksContact.pubkey;
  }
@@ -139,11 +132,6 @@ export class ComposeStorageModule extends ViewModule<ComposeView> {
  if (!contact.pubkey || !contact.fingerprint) {
  return;
  }
- if (!contact.pubkey_last_sig) {
- const lastSig = Number(contact.pubkey.lastModified);
- contact.pubkey_last_sig = lastSig;
- await ContactStore.update(undefined, contact.email, { pubkey_last_sig: lastSig });
- }
  const lastCheckOverWeekAgoOrNever = !contact.pubkey_last_check || new Date(contact.pubkey_last_check).getTime() < Date.now() - (1000 * 60 * 60 * 24 * 7);
  const isExpired = contact.expiresOn && contact.expiresOn < Date.now();
  if (lastCheckOverWeekAgoOrNever || isExpired) {
@@ -164,7 +152,7 @@ export class ComposeStorageModule extends ViewModule<ComposeView> {
  }
  }
  }
- await ContactStore.update(undefined, contact.email, { pubkey_last_check: Date.now() });
+ await ContactStore.update(undefined, contact.email, { pubkey: contact.pubkey, pubkey_last_check: Date.now() });
  // we checked for newer key and it did not result in updating the key, don't check again for another week
  } catch (e) {
  ApiErr.reportIfSignificant(e);

extension/chrome/elements/pgp_block_modules/pgp-block-signature-module.ts

@@ -55,11 +55,12 @@ export class PgpBlockViewSignatureModule {
  if (senderEmail) { // we know who sent it
  const [senderContactByEmail] = await ContactStore.get(undefined, [senderEmail]);
  if (senderContactByEmail && senderContactByEmail.pubkey) {
- render(`Fetched the right pubkey ${signerLongid} from keyserver, but will not use it because you have conflicting pubkey ${senderContactByEmail.longid} loaded.`, () => undefined);
+ const foundId = senderContactByEmail.pubkey.id;
+ render(`Fetched the right pubkey ${signerLongid} from keyserver, but will not use it because you have conflicting pubkey ${foundId} loaded.`, () => undefined);
  return;
  }
  // ---> and user doesn't have pubkey for that email addr
- const { pubkey, pgpClient } = await this.view.pubLookup.lookupEmail(senderEmail);
+ const { pubkey } = await this.view.pubLookup.lookupEmail(senderEmail);
  if (!pubkey) {
  render(`Missing pubkey ${signerLongid}`, () => undefined);
  return;
@@ -71,7 +72,7 @@ export class PgpBlockViewSignatureModule {
  return;
  }
  // ---> and longid it matches signature
- await ContactStore.save(undefined, await ContactStore.obj({ email: senderEmail, pubkey, client: pgpClient })); // <= TOFU auto-import
+ await ContactStore.save(undefined, await ContactStore.obj({ email: senderEmail, pubkey })); // <= TOFU auto-import
  render('Fetched pubkey, click to verify', () => window.location.reload());
  } else { // don't know who sent it
  render('Cannot verify: missing pubkey, missing sender info', () => undefined);

extension/chrome/elements/pgp_pubkey.ts

@@ -146,7 +146,7 @@ View.run(class PgpPubkeyView extends View {
  for (const pubkey of this.publicKeys!) {
  const email = pubkey.emails[0];
  if (email) {
- contacts.push(await ContactStore.obj({ email, client: 'pgp', pubkey: KeyUtil.armor(pubkey), lastSig: Number(pubkey.lastModified) }));
+ contacts.push(await ContactStore.obj({ email, pubkey: KeyUtil.armor(pubkey) }));
  }
  }
  await ContactStore.save(undefined, contacts);
@@ -158,12 +158,7 @@ View.run(class PgpPubkeyView extends View {
  $('.input_email').remove();
  } else if (this.publicKeys!.length) {
  if (Str.isEmailValid(String($('.input_email').val()))) {
- const contact = await ContactStore.obj({
- email: String($('.input_email').val()),
- client: 'pgp',
- pubkey: KeyUtil.armor(this.publicKeys![0]),
- lastSig: Number(this.publicKeys![0].lastModified)
- });
+ const contact = await ContactStore.obj({ email: String($('.input_email').val()), pubkey: KeyUtil.armor(this.publicKeys![0]) });
  await ContactStore.save(undefined, contact);
  BrowserMsg.send.addToContacts(this.parentTabId);
  Xss.sanitizeReplace(addContactBtn, `<span class="good">${Xss.escape(String($('.input_email').val()))} added</span>`);

extension/chrome/settings/modules/contacts.ts

@@ -117,12 +117,7 @@ View.run(class ContactsView extends View {
  private actionRenderViewPublicKeyHandler = async (viewPubkeyButton: HTMLElement) => {
  const [contact] = await ContactStore.get(undefined, [$(viewPubkeyButton).closest('tr').attr('email')!]); // defined above
  $('.hide_when_rendering_subpage').css('display', 'none');
- Xss.sanitizeRender('h1', `${this.backBtn}${this.space}${contact!.email}`); // should exist - from list of contacts
- if (contact!.client === 'cryptup') {
- Xss.sanitizeAppend('h1', '&nbsp;&nbsp;&nbsp;&nbsp;<img src="/img/logo/flowcrypt-logo-19-19.png" />');
- } else {
- Xss.sanitizeAppend('h1', '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;');
- }
+ Xss.sanitizeRender('h1', `${this.backBtn}${this.space}${contact!.email}&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`); // should exist - from list of contacts
  $('#view_contact .key_dump').text(KeyUtil.armor(contact!.pubkey!)); // should exist - from list of contacts && should have pgp - filtered
  $('#view_contact #container-pubkey-details').text([
  `Type: ${contact?.pubkey?.type}`,

extension/chrome/settings/setup.ts

@@ -211,13 +211,7 @@ export class SetupView extends View {
  const myOwnEmailAddrsAsContacts: Contact[] = [];
  const { full_name: name } = await AcctStore.get(this.acctEmail, ['full_name']);
  for (const email of this.submitKeyForAddrs) {
- myOwnEmailAddrsAsContacts.push(await ContactStore.obj({
- email,
- name,
- client: 'cryptup',
- pubkey: KeyUtil.armor(await KeyUtil.asPublicKey(prvs[0])),
- lastSig: Number(prvs[0].lastModified)
- }));
+ myOwnEmailAddrsAsContacts.push(await ContactStore.obj({ email, name, pubkey: KeyUtil.armor(await KeyUtil.asPublicKey(prvs[0])) }));
  }
  await ContactStore.save(undefined, myOwnEmailAddrsAsContacts);
  }

extension/js/background_page/background_page.ts

@@ -9,7 +9,7 @@ import { Catch } from '../common/platform/catch.js';
 import { GoogleAuth } from '../common/api/email-provider/gmail/google-auth.js';
 import { VERSION } from '../common/core/const.js';
 import { injectFcIntoWebmail } from './inject.js';
-import { migrateGlobal } from './migrations.js';
+import { migrateGlobal, moveContactsToEmailsAndPubkeys } from './migrations.js';
 import { opgp } from '../common/core/crypto/pgp/openpgpjs-custom.js';
 import { GlobalStoreDict, GlobalStore } from '../common/platform/store/global-store.js';
 import { ContactStore } from '../common/platform/store/contact-store.js';
@@ -41,6 +41,7 @@ opgp.initWorker({ path: '/lib/openpgp.worker.js' });
 
  try {
  db = await ContactStore.dbOpen(); // takes 4-10 ms first time
+ await moveContactsToEmailsAndPubkeys(db);
  } catch (e) {
  await BgUtils.handleStoreErr(e);
  return;

extension/js/background_page/migrations.ts

@@ -2,10 +2,23 @@
 
 'use strict';
 
-import { KeyInfo, KeyUtil } from '../common/core/crypto/key.js';
+import { Key, KeyInfo, KeyUtil } from '../common/core/crypto/key.js';
+import { ContactStore, ContactUpdate } from '../common/platform/store/contact-store.js';
 import { GlobalStore } from '../common/platform/store/global-store.js';
 import { KeyStore } from '../common/platform/store/key-store.js';
 
+// contact entity prior to version 4
+type ContactV3 = {
+ email: string;
+ name: string | null;
+ pubkey: Key | string | null;
+ has_pgp: 0 | 1;
+ fingerprint: string | null;
+ last_use: number | null;
+ pubkey_last_check: number | null;
+ expiresOn: number | null;
+};
+
 const addKeyInfoFingerprints = async () => {
  for (const acctEmail of await GlobalStore.acctEmailsGet()) {
  const originalKis = await KeyStore.get(acctEmail);
@@ -26,3 +39,65 @@ export const migrateGlobal = async () => {
  console.info('done migrating');
  }
 };
+
+export const moveContactsToEmailsAndPubkeys = async (db: IDBDatabase): Promise<void> => {
+ if (!db.objectStoreNames.contains('contacts')) {
+ return;
+ }
+ console.info('migrating contacts of ContactStore to emails and pubkeys...');
+ const batchSize = 50;
+ try {
+ while (await moveContactsBatchToEmailsAndPubkeys(db, batchSize)) {
+ console.info('proceeding to the next batch');
+ }
+ console.info('migrating contacts of ContactStore is complete');
+ } catch (e) {
+ console.error(`Error happened when converting contacts: ${e instanceof Error ? e.message : String(e)}`);
+ }
+};
+
+const moveContactsBatchToEmailsAndPubkeys = async (db: IDBDatabase, count?: number | undefined): Promise<number> => {
+ const entries: ContactV3[] = [];
+ {
+ const tx = db.transaction(['contacts'], 'readonly');
+ await new Promise((resolve, reject) => {
+ ContactStore.setTxHandlers(tx, resolve, reject);
+ const contacts = tx.objectStore('contacts');
+ const search = contacts.getAll(undefined, count);
+ ContactStore.setReqPipe(search, (result: ContactV3[]) => {
+ entries.push(...result);
+ });
+ });
+ if (!entries.length) {
+ return 0;
+ }
+ }
+ console.info(`Processing a batch of ${entries.length}.`);
+ // transform
+ const converted = await Promise.all(entries.map(async (entry) => {
+ const armoredPubkey = (entry.pubkey && typeof entry.pubkey === 'object')
+ ? KeyUtil.armor(entry.pubkey as Key) : entry.pubkey as string;
+ // parse again to re-calculate expiration-related fields etc.
+ const pubkey = armoredPubkey ? await KeyUtil.parse(armoredPubkey) : undefined;
+ return {
+ email: entry.email,
+ update: {
+ name: entry.name,
+ pubkey,
+ last_use: entry.last_use,
+ pubkey_last_check: pubkey ? entry.pubkey_last_check : undefined
+ } as ContactUpdate
+ };
+ }));
+ {
+ const tx = db.transaction(['contacts', 'emails', 'pubkeys'], 'readwrite');
+ await new Promise((resolve, reject) => {
+ ContactStore.setTxHandlers(tx, resolve, reject);
+ for (const item of converted) {
+ ContactStore.updateTx(tx, item.email, item.update);
+ tx.objectStore('contacts').delete(item.email);
+ }
+ });
+ }
+ return converted.length;
+};

extension/js/common/core/common.ts

@@ -168,6 +168,19 @@ export class Str {
 
 }
 
+export class DateUtility {
+ // tslint:disable-next-line:no-null-keyword
+ public static asNumber = (date: number | null | undefined): number | null => {
+ if (typeof date === 'number') {
+ return date;
+ } else if (!date) {
+ return null; // tslint:disable-line:no-null-keyword
+ } else {
+ return new Date(date).getTime();
+ }
+ }
+}
+
 export class Value {
 
  public static arr = {

extension/js/common/core/crypto/key.ts

@@ -50,16 +50,10 @@ export type PubkeyResult = { pubkey: Key, email: string, isMine: boolean };
 export type Contact = {
  email: string;
  name: string | null;
- pubkey: Key | null;
+ pubkey: Key | undefined;
  has_pgp: 0 | 1;
- searchable: string[];
- client: string | null;
  fingerprint: string | null;
- longid: string | null;
- longids: string[];
- pending_lookup: number;
  last_use: number | null;
- pubkey_last_sig: number | null;
  pubkey_last_check: number | null;
  expiresOn: number | null;
 };

extension/js/common/core/crypto/smime/smime-key.ts

@@ -56,8 +56,8 @@ export class SmimeKey {
  expiration: SmimeKey.dateToNumber(certificate.validity.notAfter),
  fullyDecrypted: true,
  fullyEncrypted: false,
- isPublic: false, // the way isPublic is currently used is as opposite of isPrivate, even if the Key contains both
- isPrivate: true,
+ isPublic: certificate.publicKey && !certificate.privateKey,
+ isPrivate: !!certificate.privateKey,
  } as Key;
  const headers = PgpArmor.headers('pkcs12');
  (key as unknown as { raw: string }).raw = `${headers.begin}\n${forge.util.encode64(bytes)}\n${headers.end}`;
@@ -107,8 +107,8 @@ export class SmimeKey {
  expiration: SmimeKey.dateToNumber(certificate.validity.notAfter),
  fullyDecrypted: false,
  fullyEncrypted: false,
- isPublic: true,
- isPrivate: true,
+ isPublic: certificate.publicKey && !certificate.privateKey,
+ isPrivate: !!certificate.privateKey,
  } as Key;
  (key as unknown as { rawArmored: string }).rawArmored = text;
  return key;

extension/js/common/platform/store/abstract-store.ts

@@ -60,6 +60,15 @@ export abstract class AbstractStore {
  }
  }
 
+ public static setReqOnError = (req: IDBRequest | IDBTransaction, reject: (reason?: any) => void) => {
+ req.onerror = () => reject(AbstractStore.errCategorize(req.error || new Error('Unknown db error')));
+ }
+
+ public static setTxHandlers = (tx: IDBTransaction, resolve: (value: unknown) => void, reject: (reason?: any) => void) => {
+ tx.oncomplete = () => resolve(undefined);
+ AbstractStore.setReqOnError(tx, reject);
+ }
+
  protected static buildSingleAccountStoreFromRawResults = (scope: string, storageObj: RawStore): AcctStoreDict => {
  const accountStore: AcctStoreDict = {};
  for (const k of Object.keys(storageObj)) {