Optimalization of RebuildMailboxes
This commit is contained in:
parent
99721b6577
commit
62ade99e37
|
@ -6,6 +6,7 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/)
|
|||
|
||||
|
||||
### Changed
|
||||
* GODT-165 Optimalization of RebuildMailboxes
|
||||
* Adding DSN Sentry as build time parameter
|
||||
|
||||
## [v1.2.6] Donghai - beta (2020-03-XXX)
|
||||
|
|
2
go.sum
2
go.sum
|
@ -169,6 +169,8 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H
|
|||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/therecipe/qt v0.0.0-20200126204426-5074eb6d8c41 h1:yBVcrpbaQYJBdKT2pxTdlL4hBE/eM4UPcyj9YpyvSok=
|
||||
github.com/therecipe/qt v0.0.0-20200126204426-5074eb6d8c41/go.mod h1:SUUR2j3aE1z6/g76SdD6NwACEpvCxb3fvG82eKbD6us=
|
||||
github.com/therecipe/qt/internal/binding/files/docs/5.12.0 v0.0.0-20200126204426-5074eb6d8c41/go.mod h1:7m8PDYDEtEVqfjoUQc2UrFqhG0CDmoVJjRlQxexndFc=
|
||||
github.com/therecipe/qt/internal/binding/files/docs/5.13.0 v0.0.0-20200126204426-5074eb6d8c41/go.mod h1:mH55Ek7AZcdns5KPp99O0bg+78el64YCYWHiQKrOdt4=
|
||||
github.com/twinj/uuid v1.0.0 h1:fzz7COZnDrXGTAOHGuUGYd6sG+JMq+AoE7+Jlu0przk=
|
||||
github.com/twinj/uuid v1.0.0/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY=
|
||||
github.com/urfave/cli v1.22.3 h1:FpNT6zq26xNpHZy08emi755QwzLPs6Pukqjlc7RfOMU=
|
||||
|
|
|
@ -71,7 +71,6 @@ func TestUserSwitchAddressMode(t *testing.T) {
|
|||
m.credentialsStore.EXPECT().SwitchAddressMode("user").Return(nil),
|
||||
m.credentialsStore.EXPECT().Get("user").Return(testCredentialsSplit, nil),
|
||||
)
|
||||
m.pmapiClient.EXPECT().ListMessages(gomock.Any()).Return([]*pmapi.Message{}, 0, nil)
|
||||
|
||||
assert.NoError(t, user.SwitchAddressMode())
|
||||
assert.False(t, user.store.IsCombinedMode())
|
||||
|
|
|
@ -115,7 +115,7 @@ func syncDraftsIfNecssary(mb *Mailbox) { //nolint[funlen]
|
|||
}
|
||||
for _, msgLabelID := range msg.LabelIDs {
|
||||
if msgLabelID == pmapi.DraftLabel {
|
||||
log.WithField("id", msg.ID).Debug("Drafts mailbox created: syncing draft locally")
|
||||
log.WithField("id", msg.ID).Trace("Drafts mailbox created: syncing draft locally")
|
||||
_ = mb.txCreateOrUpdateMessages(tx, []*pmapi.Message{msg})
|
||||
break
|
||||
}
|
||||
|
|
|
@ -69,8 +69,7 @@ func (store *Store) RebuildMailboxes() (err error) {
|
|||
}
|
||||
|
||||
log.WithField("user", store.UserID()).Trace("Rebuilding mailboxes")
|
||||
store.triggerSync()
|
||||
return nil
|
||||
return store.initMailboxesBucket()
|
||||
}
|
||||
|
||||
// createOrDeleteAddressesEvent creates address objects in the store for each necessary address
|
||||
|
@ -186,31 +185,48 @@ func (store *Store) truncateMailboxesBucket() (err error) {
|
|||
}
|
||||
|
||||
// initMailboxesBucket recreates the mailboxes bucket from the metadata bucket.
|
||||
func (store *Store) initMailboxesBucket() error { //nolint[unused]
|
||||
i := 0
|
||||
tx := func(tx *bolt.Tx) error {
|
||||
return tx.Bucket(metadataBucket).ForEach(func(k, v []byte) error {
|
||||
func (store *Store) initMailboxesBucket() error {
|
||||
return store.db.Update(func(tx *bolt.Tx) error {
|
||||
i := 0
|
||||
msgs := []*pmapi.Message{}
|
||||
|
||||
err := tx.Bucket(metadataBucket).ForEach(func(k, v []byte) error {
|
||||
msg := &pmapi.Message{}
|
||||
|
||||
if err := json.Unmarshal(v, msg); err != nil {
|
||||
return err
|
||||
}
|
||||
msgs = append(msgs, msg)
|
||||
|
||||
for _, a := range store.addresses {
|
||||
if err := a.txCreateOrUpdateMessages(tx, []*pmapi.Message{msg}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Calling txCreateOrUpdateMessages does some overhead by iterating
|
||||
// all mailboxes, accessing buckets and so on. It's better to do in
|
||||
// batches instead of one by one (seconds vs hours for huge accounts).
|
||||
// Average size of metadata is 1k bytes, sometimes up to 2k bytes.
|
||||
// 10k messages will take about 20 MB of memory.
|
||||
i++
|
||||
if i%100 == 0 {
|
||||
store.log.WithField("i", i).
|
||||
Trace("Init mboxes heartbeat")
|
||||
if i%10000 == 0 {
|
||||
store.log.WithField("i", i).Debug("Init mboxes heartbeat")
|
||||
|
||||
for _, a := range store.addresses {
|
||||
if err := a.txCreateOrUpdateMessages(tx, msgs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
msgs = []*pmapi.Message{}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return store.db.Update(tx)
|
||||
for _, a := range store.addresses {
|
||||
if err := a.txCreateOrUpdateMessages(tx, msgs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue