Skip to content

Commit 9c7cb48

Browse files
LBeernaertProtonLBeernaertProton
committed
fix(GODT-1757): Recent flag behavior and snapshot db reads
This patch removes the need for the snapshots to read new messages from the database when they arrive via an `exists` responder. We have also updated the behavior of the `Recent` flag to match dovecot's implementation. If a client append a message to a selected mailbox only that client will receive the recent flags. If a client appends a message to a non-selected mailbox, the first state which receives the update will apply the recent flag. To achieve the latter, we added a new update `ExistsStateUpdate` which enforces the logic described above in a thread safe manner.
1 parent 81b1a3a commit 9c7cb48

14 files changed

Lines changed: 299 additions & 98 deletions

‎internal/backend/backend.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ func (b *Backend) GetState(username, password string, sessionID int) (*state.Sta
120120
logrus.
121121
WithField("userID", userID).
122122
WithField("username", username).
123+
WithField("stateID", state.StateID).
123124
Debug("Created new IMAP state")
124125

125126
return state, nil

‎internal/backend/connector_updates.go‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -223,11 +223,8 @@ func (user *user) applyMessagesCreated(ctx context.Context, update *imap.Message
223223
return err
224224
}
225225

226-
responders := xslices.Map(messageIDs, func(messageID ids.MessageIDPair) state.Responder {
227-
return state.NewExists(messageID.InternalID, messageUIDs[messageID.InternalID])
228-
})
226+
user.queueStateUpdate(state.NewExistsStateUpdate(internalMailboxID, messageIDs, messageUIDs, nil))
229227

230-
user.queueStateUpdate(state.NewMailboxIDResponderStateUpdate(internalMailboxID, responders...))
231228
}
232229

233230
return nil
@@ -347,8 +344,8 @@ func (user *user) setMessageMailboxes(ctx context.Context, tx *ent.Tx, messageID
347344
}
348345

349346
// applyMessagesAddedToMailbox adds the messages to the given mailbox.
350-
func (user *user) applyMessagesAddedToMailbox(ctx context.Context, tx *ent.Tx, mboxID imap.InternalMailboxID, messageIDs []imap.InternalMessageID) (map[imap.InternalMessageID]int, error) {
351-
messageUIDs, update, err := state.AddMessagesToMailbox(ctx, tx, mboxID, messageIDs)
347+
func (user *user) applyMessagesAddedToMailbox(ctx context.Context, tx *ent.Tx, mboxID imap.InternalMailboxID, messageIDs []imap.InternalMessageID) (map[imap.InternalMessageID]*ent.UID, error) {
348+
messageUIDs, update, err := state.AddMessagesToMailbox(ctx, tx, mboxID, messageIDs, nil)
352349
if err != nil {
353350
return nil, err
354351
}

‎internal/db/message.go‎

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ func CreateMessages(ctx context.Context, tx *ent.Tx, reqs ...*CreateMessageReq)
5555
})...).Save(ctx)
5656
}
5757

58-
func AddMessagesToMailbox(ctx context.Context, tx *ent.Tx, messageIDs []imap.InternalMessageID, mboxID imap.InternalMailboxID) (map[imap.InternalMessageID]int, error) {
58+
func AddMessagesToMailbox(ctx context.Context, tx *ent.Tx, messageIDs []imap.InternalMessageID, mboxID imap.InternalMailboxID) (map[imap.InternalMessageID]*ent.UID, error) {
5959
messageUIDs := make(map[imap.InternalMessageID]int)
6060

6161
mbox, err := tx.Mailbox.Query().Where(mailbox.MailboxID(mboxID)).Only(ctx)
@@ -88,10 +88,10 @@ func AddMessagesToMailbox(ctx context.Context, tx *ent.Tx, messageIDs []imap.Int
8888
return nil, err
8989
}
9090

91-
return messageUIDs, nil
91+
return GetMessageUIDsWithFlags(ctx, tx.Client(), mboxID, messageIDs)
9292
}
9393

94-
func BumpMailboxUIDsForMessage(ctx context.Context, tx *ent.Tx, messageIDs []imap.InternalMessageID, mboxID imap.InternalMailboxID) (map[imap.InternalMessageID]int, error) {
94+
func BumpMailboxUIDsForMessage(ctx context.Context, tx *ent.Tx, messageIDs []imap.InternalMessageID, mboxID imap.InternalMailboxID) (map[imap.InternalMessageID]*ent.UID, error) {
9595
messageUIDs := make(map[imap.InternalMessageID]int)
9696

9797
mbox, err := tx.Mailbox.Query().Where(mailbox.MailboxID(mboxID)).Only(ctx)
@@ -121,7 +121,7 @@ func BumpMailboxUIDsForMessage(ctx context.Context, tx *ent.Tx, messageIDs []ima
121121
return nil, err
122122
}
123123

124-
return messageUIDs, nil
124+
return GetMessageUIDsWithFlags(ctx, tx.Client(), mboxID, messageIDs)
125125
}
126126

127127
func RemoveMessagesFromMailbox(ctx context.Context, tx *ent.Tx, messageIDs []imap.InternalMessageID, mboxID imap.InternalMailboxID) error {
@@ -207,6 +207,7 @@ func GetMessageUIDs(ctx context.Context, client *ent.Client, mboxID imap.Interna
207207
).
208208
WithMessage(func(query *ent.MessageQuery) {
209209
query.Select(message.FieldMessageID)
210+
query.Select(message.FieldRemoteID)
210211
}).
211212
All(ctx)
212213
if err != nil {
@@ -222,6 +223,32 @@ func GetMessageUIDs(ctx context.Context, client *ent.Client, mboxID imap.Interna
222223
return res, nil
223224
}
224225

226+
func GetMessageUIDsWithFlags(ctx context.Context, client *ent.Client, mboxID imap.InternalMailboxID, messageIDs []imap.InternalMessageID) (map[imap.InternalMessageID]*ent.UID, error) {
227+
messageUIDs, err := client.UID.Query().
228+
Where(
229+
uid.HasMailboxWith(mailbox.MailboxID(mboxID)),
230+
uid.HasMessageWith(message.MessageIDIn(messageIDs...)),
231+
).
232+
WithMessage(func(query *ent.MessageQuery) {
233+
query.Select(message.FieldMessageID, message.FieldRemoteID)
234+
query.WithFlags(func(query *ent.MessageFlagQuery) {
235+
query.Select(messageflag.FieldValue)
236+
})
237+
}).
238+
All(ctx)
239+
if err != nil {
240+
return nil, err
241+
}
242+
243+
res := make(map[imap.InternalMessageID]*ent.UID)
244+
245+
for _, messageUID := range messageUIDs {
246+
res[messageUID.Edges.Message.MessageID] = messageUID
247+
}
248+
249+
return res, nil
250+
}
251+
225252
// GetMessageFlags returns the flags of the given messages.
226253
// It does not include per-mailbox flags (\Deleted, \Recent)!
227254
func GetMessageFlags(ctx context.Context, client *ent.Client, messageIDs []imap.InternalMessageID) (map[imap.InternalMessageID]imap.FlagSet, error) {
@@ -467,3 +494,19 @@ func GetMessageIDFromRemoteID(ctx context.Context, client *ent.Client, id imap.M
467494

468495
return message.MessageID, nil
469496
}
497+
498+
func NewFlagSet(msgUID *ent.UID, flags []*ent.MessageFlag) imap.FlagSet {
499+
flagSet := imap.NewFlagSetFromSlice(xslices.Map(flags, func(flag *ent.MessageFlag) string {
500+
return flag.Value
501+
}))
502+
503+
if msgUID.Deleted {
504+
flagSet = flagSet.Add(imap.FlagDeleted)
505+
}
506+
507+
if msgUID.Recent {
508+
flagSet = flagSet.Add(imap.FlagRecent)
509+
}
510+
511+
return flagSet
512+
}

‎internal/session/session.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,13 @@ func (s *Session) serve(ctx context.Context) error {
139139

140140
for {
141141
select {
142+
case stateUpdate := <-s.state.GetStateUpdatesCh():
143+
if err := s.state.ApplyUpdate(ctx, stateUpdate); err != nil {
144+
logrus.WithError(err).Error("Failed to apply state update")
145+
}
146+
147+
continue
148+
142149
case res, ok := <-cmdCh:
143150
if !ok {
144151
logrus.Debugf("Failed to read from command channel")
@@ -163,13 +170,6 @@ func (s *Session) serve(ctx context.Context) error {
163170
case <-s.state.Done():
164171
return nil
165172

166-
case stateUpdate := <-s.state.GetStateUpdatesCh():
167-
if err := s.state.ApplyUpdate(ctx, stateUpdate); err != nil {
168-
logrus.WithError(err).Error("Failed to apply state update")
169-
}
170-
171-
continue
172-
173173
case <-ctx.Done():
174174
return ctx.Err()
175175
}

‎internal/state/actions.go‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ func (state *State) actionCreateMessage(
9090
literal []byte,
9191
flags imap.FlagSet,
9292
date time.Time,
93+
isSelectedMailbox bool,
9394
) (int, error) {
9495
internalID, res, err := state.user.GetRemote().CreateMessage(ctx, mboxID.RemoteID, literal, flags, date)
9596
if err != nil {
@@ -136,19 +137,35 @@ func (state *State) actionCreateMessage(
136137
return 0, err
137138
}
138139

139-
if err := state.user.QueueOrApplyStateUpdate(ctx, tx, NewMailboxIDResponderStateUpdate(mboxID.InternalID, NewExists(internalID, messageUIDs[internalID]))); err != nil {
140+
// We can append to non-selected mailboxes.
141+
var st *State
142+
if isSelectedMailbox {
143+
st = state
144+
}
145+
146+
uid := messageUIDs[internalID]
147+
if err := state.user.QueueOrApplyStateUpdate(
148+
ctx,
149+
tx,
150+
newExistsStateUpdateWithExists(
151+
mboxID.InternalID,
152+
[]*exists{newExists(ids.MessageIDPair{InternalID: internalID, RemoteID: res.ID}, uid.UID, db.NewFlagSet(uid, uid.Edges.Message.Edges.Flags))},
153+
st,
154+
),
155+
); err != nil {
140156
return 0, err
141157
}
142158

143-
return messageUIDs[internalID], nil
159+
return messageUIDs[internalID].UID, nil
144160
}
145161

146162
func (state *State) actionAddMessagesToMailbox(
147163
ctx context.Context,
148164
tx *ent.Tx,
149165
messageIDs []ids.MessageIDPair,
150166
mboxID ids.MailboxIDPair,
151-
) (map[imap.InternalMessageID]int, error) {
167+
isMailboxSelected bool,
168+
) (map[imap.InternalMessageID]*ent.UID, error) {
152169
var haveMessageIDs []ids.MessageIDPair
153170
if state.snap != nil && state.snap.mboxID.InternalID == mboxID.InternalID {
154171
haveMessageIDs = state.snap.getAllMessageIDs()
@@ -175,7 +192,13 @@ func (state *State) actionAddMessagesToMailbox(
175192
return nil, err
176193
}
177194

178-
messageUIDs, update, err := AddMessagesToMailbox(ctx, tx, mboxID.InternalID, internalIDs)
195+
// Messages can be added to a mailbox that is not selected.
196+
var st *State
197+
if isMailboxSelected {
198+
st = state
199+
}
200+
201+
messageUIDs, update, err := AddMessagesToMailbox(ctx, tx, mboxID.InternalID, internalIDs, st)
179202
if err != nil {
180203
return nil, err
181204
}
@@ -227,7 +250,7 @@ func (state *State) actionMoveMessages(
227250
tx *ent.Tx,
228251
messageIDs []ids.MessageIDPair,
229252
mboxFromID, mboxToID ids.MailboxIDPair,
230-
) (map[imap.InternalMessageID]int, error) {
253+
) (map[imap.InternalMessageID]*ent.UID, error) {
231254
if mboxFromID.InternalID == mboxToID.InternalID {
232255
internalIDs, _ := ids.SplitMessageIDPairSlice(messageIDs)
233256

@@ -272,7 +295,7 @@ func (state *State) actionMoveMessages(
272295
return nil, err
273296
}
274297

275-
messageUIDs, updates, err := MoveMessagesFromMailbox(ctx, tx, mboxFromID.InternalID, mboxToID.InternalID, internalIDs)
298+
messageUIDs, updates, err := MoveMessagesFromMailbox(ctx, tx, mboxFromID.InternalID, mboxToID.InternalID, internalIDs, state)
276299
if err != nil {
277300
return nil, err
278301
}

‎internal/state/mailbox.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -138,17 +138,17 @@ func (m *Mailbox) Append(ctx context.Context, literal []byte, flags imap.FlagSet
138138
return db.HasMessageWithID(ctx, client, msgID)
139139
}); err != nil || !exists {
140140
logrus.WithError(err).Warn("The message has an unknown internal ID")
141-
} else if res, err := db.WriteResult(ctx, m.state.db(), func(ctx context.Context, tx *ent.Tx) (map[imap.InternalMessageID]int, error) {
142-
return m.state.actionAddMessagesToMailbox(ctx, tx, []ids.MessageIDPair{ids.NewMessageIDPairWithoutRemote(msgID)}, ids.NewMailboxIDPair(m.mbox))
141+
} else if res, err := db.WriteResult(ctx, m.state.db(), func(ctx context.Context, tx *ent.Tx) (map[imap.InternalMessageID]*ent.UID, error) {
142+
return m.state.actionAddMessagesToMailbox(ctx, tx, []ids.MessageIDPair{ids.NewMessageIDPairWithoutRemote(msgID)}, ids.NewMailboxIDPair(m.mbox), m.snap == m.state.snap)
143143
}); err != nil {
144144
return 0, err
145145
} else {
146-
return res[msgID], nil
146+
return res[msgID].UID, nil
147147
}
148148
}
149149

150150
return db.WriteAndStoreResult(ctx, m.state.db(), m.state.user.GetStore(), func(ctx context.Context, tx *ent.Tx, transaction store.Transaction) (int, error) {
151-
return m.state.actionCreateMessage(ctx, tx, transaction, m.snap.mboxID, literal, flags, date)
151+
return m.state.actionCreateMessage(ctx, tx, transaction, m.snap.mboxID, literal, flags, date, m.snap == m.state.snap)
152152
})
153153
}
154154

@@ -176,8 +176,8 @@ func (m *Mailbox) Copy(ctx context.Context, seq *proto.SequenceSet, name string)
176176
return msg.UID
177177
})
178178

179-
destUIDs, err := db.WriteResult(ctx, m.state.db(), func(ctx context.Context, tx *ent.Tx) (map[imap.InternalMessageID]int, error) {
180-
return m.state.actionAddMessagesToMailbox(ctx, tx, msgIDs, ids.NewMailboxIDPair(mbox))
179+
destUIDs, err := db.WriteResult(ctx, m.state.db(), func(ctx context.Context, tx *ent.Tx) (map[imap.InternalMessageID]*ent.UID, error) {
180+
return m.state.actionAddMessagesToMailbox(ctx, tx, msgIDs, ids.NewMailboxIDPair(mbox), m.snap == m.state.snap)
181181
})
182182
if err != nil {
183183
return nil, err
@@ -187,7 +187,7 @@ func (m *Mailbox) Copy(ctx context.Context, seq *proto.SequenceSet, name string)
187187

188188
if len(destUIDs) > 0 {
189189
res = response.ItemCopyUID(mbox.UIDValidity, msgUIDs, xslices.Map(maps.Keys(destUIDs), func(messageID imap.InternalMessageID) int {
190-
return destUIDs[messageID]
190+
return destUIDs[messageID].UID
191191
}))
192192
}
193193

@@ -218,7 +218,7 @@ func (m *Mailbox) Move(ctx context.Context, seq *proto.SequenceSet, name string)
218218
return msg.UID
219219
})
220220

221-
destUIDs, err := db.WriteResult(ctx, m.state.db(), func(ctx context.Context, tx *ent.Tx) (map[imap.InternalMessageID]int, error) {
221+
destUIDs, err := db.WriteResult(ctx, m.state.db(), func(ctx context.Context, tx *ent.Tx) (map[imap.InternalMessageID]*ent.UID, error) {
222222
return m.state.actionMoveMessages(ctx, tx, msgIDs, m.snap.mboxID, ids.NewMailboxIDPair(mbox))
223223
})
224224
if err != nil {
@@ -229,7 +229,7 @@ func (m *Mailbox) Move(ctx context.Context, seq *proto.SequenceSet, name string)
229229

230230
if len(destUIDs) > 0 {
231231
res = response.ItemCopyUID(mbox.UIDValidity, msgUIDs, xslices.Map(maps.Keys(destUIDs), func(messageID imap.InternalMessageID) int {
232-
return destUIDs[messageID]
232+
return destUIDs[messageID].UID
233233
}))
234234
}
235235

0 commit comments

Comments
 (0)