diff --git a/Changelog.md b/Changelog.md index fc273061..3274c9a0 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,7 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/) ### Changed +* GODT-165 Optimalization of RebuildMailboxes * Adding DSN Sentry as build time parameter ## [v1.2.6] Donghai - beta (2020-03-XXX) diff --git a/go.sum b/go.sum index 238dd6d3..dbda013d 100644 --- a/go.sum +++ b/go.sum @@ -169,6 +169,8 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/therecipe/qt v0.0.0-20200126204426-5074eb6d8c41 h1:yBVcrpbaQYJBdKT2pxTdlL4hBE/eM4UPcyj9YpyvSok= github.com/therecipe/qt v0.0.0-20200126204426-5074eb6d8c41/go.mod h1:SUUR2j3aE1z6/g76SdD6NwACEpvCxb3fvG82eKbD6us= +github.com/therecipe/qt/internal/binding/files/docs/5.12.0 v0.0.0-20200126204426-5074eb6d8c41/go.mod h1:7m8PDYDEtEVqfjoUQc2UrFqhG0CDmoVJjRlQxexndFc= +github.com/therecipe/qt/internal/binding/files/docs/5.13.0 v0.0.0-20200126204426-5074eb6d8c41/go.mod h1:mH55Ek7AZcdns5KPp99O0bg+78el64YCYWHiQKrOdt4= github.com/twinj/uuid v1.0.0 h1:fzz7COZnDrXGTAOHGuUGYd6sG+JMq+AoE7+Jlu0przk= github.com/twinj/uuid v1.0.0/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY= github.com/urfave/cli v1.22.3 h1:FpNT6zq26xNpHZy08emi755QwzLPs6Pukqjlc7RfOMU= diff --git a/internal/bridge/user_credentials_test.go b/internal/bridge/user_credentials_test.go index 20268f11..04cfe322 100644 --- a/internal/bridge/user_credentials_test.go +++ b/internal/bridge/user_credentials_test.go @@ -71,7 +71,6 @@ func TestUserSwitchAddressMode(t *testing.T) { m.credentialsStore.EXPECT().SwitchAddressMode("user").Return(nil), m.credentialsStore.EXPECT().Get("user").Return(testCredentialsSplit, nil), ) - m.pmapiClient.EXPECT().ListMessages(gomock.Any()).Return([]*pmapi.Message{}, 0, nil) assert.NoError(t, user.SwitchAddressMode()) assert.False(t, user.store.IsCombinedMode()) diff --git a/internal/store/mailbox.go b/internal/store/mailbox.go index c6232765..420792c9 100644 --- a/internal/store/mailbox.go +++ b/internal/store/mailbox.go @@ -115,7 +115,7 @@ func syncDraftsIfNecssary(mb *Mailbox) { //nolint[funlen] } for _, msgLabelID := range msg.LabelIDs { if msgLabelID == pmapi.DraftLabel { - log.WithField("id", msg.ID).Debug("Drafts mailbox created: syncing draft locally") + log.WithField("id", msg.ID).Trace("Drafts mailbox created: syncing draft locally") _ = mb.txCreateOrUpdateMessages(tx, []*pmapi.Message{msg}) break } diff --git a/internal/store/user_address.go b/internal/store/user_address.go index 811e3e4b..5bc2e6b8 100644 --- a/internal/store/user_address.go +++ b/internal/store/user_address.go @@ -69,8 +69,7 @@ func (store *Store) RebuildMailboxes() (err error) { } log.WithField("user", store.UserID()).Trace("Rebuilding mailboxes") - store.triggerSync() - return nil + return store.initMailboxesBucket() } // createOrDeleteAddressesEvent creates address objects in the store for each necessary address @@ -186,31 +185,48 @@ func (store *Store) truncateMailboxesBucket() (err error) { } // initMailboxesBucket recreates the mailboxes bucket from the metadata bucket. -func (store *Store) initMailboxesBucket() error { //nolint[unused] - i := 0 - tx := func(tx *bolt.Tx) error { - return tx.Bucket(metadataBucket).ForEach(func(k, v []byte) error { +func (store *Store) initMailboxesBucket() error { + return store.db.Update(func(tx *bolt.Tx) error { + i := 0 + msgs := []*pmapi.Message{} + + err := tx.Bucket(metadataBucket).ForEach(func(k, v []byte) error { msg := &pmapi.Message{} if err := json.Unmarshal(v, msg); err != nil { return err } + msgs = append(msgs, msg) - for _, a := range store.addresses { - if err := a.txCreateOrUpdateMessages(tx, []*pmapi.Message{msg}); err != nil { - return err - } - } - + // Calling txCreateOrUpdateMessages does some overhead by iterating + // all mailboxes, accessing buckets and so on. It's better to do in + // batches instead of one by one (seconds vs hours for huge accounts). + // Average size of metadata is 1k bytes, sometimes up to 2k bytes. + // 10k messages will take about 20 MB of memory. i++ - if i%100 == 0 { - store.log.WithField("i", i). - Trace("Init mboxes heartbeat") + if i%10000 == 0 { + store.log.WithField("i", i).Debug("Init mboxes heartbeat") + + for _, a := range store.addresses { + if err := a.txCreateOrUpdateMessages(tx, msgs); err != nil { + return err + } + } + msgs = []*pmapi.Message{} } return nil }) - } + if err != nil { + return err + } - return store.db.Update(tx) + for _, a := range store.addresses { + if err := a.txCreateOrUpdateMessages(tx, msgs); err != nil { + return err + } + } + + return nil + }) }