nextcloud
/
desktop
mirror of https://github.com/nextcloud/desktop
0
0
Fork 0
Code Issues Releases Wiki Activity

FolderMan: Document and clean up folder scheduling

This commit is contained in:
Christian Kamm 2016-10-18 16:04:25 +02:00 committed by ckamm
parent fe984b61d7
commit a2222228c9
4 changed files with 172 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
src/gui/folder.cpp
View File

@ -58,7 +58,6 @@ Folder::Folder(const FolderDefinition& definition,
, _wipeDb(false)
, _proxyDirty(true)
, _lastSyncDuration(0)
, _forceSyncOnPollTimeout(false)
, _consecutiveFailingSyncs(0)
, _consecutiveFollowUpSyncs(0)
, _journal(definition.localPath)
@ -112,6 +111,11 @@ Folder::Folder(const FolderDefinition& definition,
connect(_engine.data(), SIGNAL(seenLockedFile(QString)), FolderMan::instance(), SLOT(slotSyncOnceFileUnlocks(QString)));
connect(_engine.data(), SIGNAL(aboutToPropagate(SyncFileItemVector&)),
SLOT(slotLogPropagationStart()));
_scheduleSelfTimer.setSingleShot(true);
_scheduleSelfTimer.setInterval(SyncEngine::minimumFileAgeForUpload);
connect(&_scheduleSelfTimer, SIGNAL(timeout()),
SLOT(slotScheduleThisFolder()));
}
Folder::~Folder()
@ -275,7 +279,7 @@ void Folder::slotRunEtagJob()
AccountPtr account = _accountState->account();
if (!_requestEtagJob.isNull()) {
if (_requestEtagJob) {
qDebug() << Q_FUNC_INFO << remoteUrl().toString() << "has ETag job queued, not trying to sync";
return;
}
@ -285,47 +289,15 @@ void Folder::slotRunEtagJob()
return;
}
bool forceSyncIntervalExpired =
quint64(_timeSinceLastSyncDone.elapsed()) > ConfigFile().forceSyncInterval();
bool syncAgainAfterFail = _consecutiveFailingSyncs > 0 && _consecutiveFailingSyncs < 3;
// Do the ordinary etag check for the root folder and schedule a
// sync if it's different.
// There are several conditions under which we trigger a full-discovery sync:
// * When a suitably long time has passed since the last sync finished
// * When the last sync failed (only a couple of times)
// * When the last sync requested another sync to be done (only a couple of times)
//
// Note that the etag check (see below) and the file watcher may also trigger
// syncs.
if (forceSyncIntervalExpired
|| _forceSyncOnPollTimeout
|| syncAgainAfterFail) {
if (forceSyncIntervalExpired) {
qDebug() << "** Force Sync, because it has been " << _timeSinceLastSyncDone.elapsed() << "ms "
<< "since the last sync";
}
if (_forceSyncOnPollTimeout) {
qDebug() << "** Force Sync, because it was requested";
}
if (syncAgainAfterFail) {
qDebug() << "** Force Sync, because the last"
<< _consecutiveFailingSyncs << "syncs failed, last status:"
<< _syncResult.statusString();
}
_forceSyncOnPollTimeout = false;
emit scheduleToSync(this);
} else {
// Do the ordinary etag check for the root folder and only schedule a real
// sync if it's different.
_requestEtagJob = new RequestEtagJob(account, remotePath(), this);
_requestEtagJob->setTimeout(60*1000);
// check if the etag is different
QObject::connect(_requestEtagJob, SIGNAL(etagRetreived(QString)), this, SLOT(etagRetreived(QString)));
FolderMan::instance()->slotScheduleETagJob(alias(), _requestEtagJob);
// The _requestEtagJob is auto deleting itself on finish. Our guard pointer _requestEtagJob will then be null.
}
_requestEtagJob = new RequestEtagJob(account, remotePath(), this);
_requestEtagJob->setTimeout(60*1000);
// check if the etag is different when retrieved
QObject::connect(_requestEtagJob, SIGNAL(etagRetreived(QString)), this, SLOT(etagRetreived(QString)));
FolderMan::instance()->slotScheduleETagJob(alias(), _requestEtagJob);
// The _requestEtagJob is auto deleting itself on finish. Our guard pointer _requestEtagJob will then be null.
}
void Folder::etagRetreived(const QString& etag)
@ -338,7 +310,7 @@ void Folder::etagRetreived(const QString& etag)
if (_lastEtag != etag) {
qDebug() << "* Compare etag with previous etag: last:" << _lastEtag << ", received:" << etag << "-> CHANGED";
_lastEtag = etag;
emit scheduleToSync(this);
slotScheduleThisFolder();
}
_accountState->tagLastSuccessfullETagRequest();
@ -598,7 +570,10 @@ void Folder::slotWatchedPathChanged(const QString& path)
}
emit watchedFileChangedExternally(path);
emit scheduleToSync(this);
// Also schedule this folder for a sync, but only after some delay:
// The sync will not upload files that were changed too recently.
scheduleThisFolderSoon();
}
void Folder::slotThreadTreeWalkResult(const SyncFileItemVector& items)
@ -879,11 +854,10 @@ void Folder::slotSyncFinished(bool success)
// Maybe force a follow-up sync to take place, but only a couple of times.
if (anotherSyncNeeded && _consecutiveFollowUpSyncs <= 3)
{
_forceSyncOnPollTimeout = true;
// We will make sure that the poll timer occurs soon enough.
// delay 1s, 4s, 9s
int c = _consecutiveFollowUpSyncs;
QTimer::singleShot(c*c * 1000, this, SLOT(slotRunEtagJob() ));
// Sometimes another sync is requested because a local file is still
// changing, so wait at least a small amount of time before syncing
// the folder again.
scheduleThisFolderSoon();
}
}
@ -964,7 +938,17 @@ void Folder::slotLogPropagationStart()
_fileLog->logLap("Propagation starts");
}
void Folder::slotScheduleThisFolder()
{
FolderMan::instance()->slotScheduleSync(this);
}
void Folder::scheduleThisFolderSoon()
{
if (!_scheduleSelfTimer.isActive()) {
_scheduleSelfTimer.start();
}
}
void Folder::slotAboutToRemoveAllFiles(SyncFileItem::Direction, bool *cancel)
{
@ -988,10 +972,8 @@ void Folder::slotAboutToRemoveAllFiles(SyncFileItem::Direction, bool *cancel)
*cancel = msgBox.clickedButton() == keepBtn;
if (*cancel) {
wipe();
// speed up next sync
_lastEtag.clear();
_forceSyncOnPollTimeout = true;
QTimer::singleShot(50, this, SLOT(slotRunEtagJob()));
slotScheduleThisFolder();
}
}

21
src/gui/folder.h
View File

@ -178,6 +178,7 @@ public:
qint64 msecSinceLastSync() const { return _timeSinceLastSyncDone.elapsed(); }
qint64 msecLastSyncDuration() const { return _lastSyncDuration; }
int consecutiveFollowUpSyncs() const { return _consecutiveFollowUpSyncs; }
int consecutiveFailingSyncs() const { return _consecutiveFailingSyncs; }
/// Saves the folder data in the account's settings.
void saveToSettings() const;
@ -194,11 +195,21 @@ public:
*/
bool isFileExcludedRelative(const QString& relativePath) const;
/** Calls schedules this folder on the FolderMan after a short delay.
*
* This should be used in situations where a sync should be triggered
* because a local file was modified. Syncs don't upload files that were
* modified too recently, and this delay ensures the modification is
* far enough in the past.
*
* The delay doesn't reset with subsequent calls.
*/
void scheduleThisFolderSoon();
signals:
void syncStateChange();
void syncStarted();
void syncFinished(const SyncResult &result);
void scheduleToSync(Folder*);
void progressInfo(const ProgressInfo& progress);
void newBigFolderDiscovered(const QString &); // A new folder bigger than the threshold was discovered
void syncPausedChanged(Folder*, bool paused);
@ -266,6 +277,11 @@ private slots:
void slotLogPropagationStart();
/** Adds this folder to the list of scheduled folders in the
* FolderMan.
*/
void slotScheduleThisFolder();
private:
bool setIgnoredFiles();
@ -302,7 +318,6 @@ private:
QElapsedTimer _timeSinceLastSyncDone;
QElapsedTimer _timeSinceLastSyncStart;
qint64 _lastSyncDuration;
bool _forceSyncOnPollTimeout;
/// The number of syncs that failed in a row.
/// Reset when a sync is successful.
@ -317,6 +332,8 @@ private:
ClientProxy _clientProxy;
QScopedPointer<SyncRunFileLog> _fileLog;
QTimer _scheduleSelfTimer;
};
}

104
src/gui/folderman.cpp
View File

@ -64,11 +64,17 @@ FolderMan::FolderMan(QObject *parent) :
connect(&_startScheduledSyncTimer, SIGNAL(timeout()),
SLOT(slotStartScheduledFolderSync()));
_timeScheduler.setInterval(5000);
_timeScheduler.setSingleShot(false);
connect(&_timeScheduler, SIGNAL(timeout()),
SLOT(slotScheduleFolderByTime()));
_timeScheduler.start();
connect(AccountManager::instance(), SIGNAL(accountRemoved(AccountState*)),
SLOT(slotRemoveFoldersForAccount(AccountState*)));
connect(_lockWatcher.data(), SIGNAL(fileUnlocked(QString)),
SLOT(slotScheduleFolderOwningFile(QString)));
SLOT(slotWatchedFileUnlocked(QString)));
}
FolderMan *FolderMan::instance()
@ -100,8 +106,6 @@ void FolderMan::unloadFolder( Folder *f )
}
_folderMap.remove( f->alias() );
disconnect(f, SIGNAL(scheduleToSync(Folder*)),
this, SLOT(slotScheduleSync(Folder*)));
disconnect(f, SIGNAL(syncStarted()),
this, SLOT(slotFolderSyncStarted()));
disconnect(f, SIGNAL(syncFinished(SyncResult)),
@ -131,7 +135,7 @@ int FolderMan::unloadAndDeleteAllFolders()
}
_lastSyncFolder = 0;
_currentSyncFolder = 0;
_scheduleQueue.clear();
_scheduledFolders.clear();
emit scheduleQueueChanged();
Q_ASSERT(_folderMap.count() == 0);
@ -496,7 +500,7 @@ void FolderMan::slotScheduleSync( Folder *f )
qDebug() << "Schedule folder " << alias << " to sync!";
if( ! _scheduleQueue.contains(f) ) {
if( ! _scheduledFolders.contains(f) ) {
if( !f->canSync() ) {
qDebug() << "Folder is not ready to sync, not scheduled!";
_socketApi->slotUpdateFolderView(f);
@ -504,7 +508,7 @@ void FolderMan::slotScheduleSync( Folder *f )
}
f->prepareToSync();
emit folderSyncStateChange(f);
_scheduleQueue.enqueue(f);
_scheduledFolders.enqueue(f);
emit scheduleQueueChanged();
} else {
qDebug() << " II> Sync for folder " << alias << " already scheduled, do not enqueue!";
@ -580,7 +584,7 @@ void FolderMan::slotAccountStateChanged()
_currentSyncFolder->slotTerminateSync();
}
QMutableListIterator<Folder*> it(_scheduleQueue);
QMutableListIterator<Folder*> it(_scheduledFolders);
while (it.hasNext()) {
Folder* f = it.next();
if (f->accountState() == accountState) {
@ -595,7 +599,7 @@ void FolderMan::slotAccountStateChanged()
// this is not the same as Pause and Resume of folders.
void FolderMan::setSyncEnabled( bool enabled )
{
if (!_syncEnabled && enabled && !_scheduleQueue.isEmpty()) {
if (!_syncEnabled && enabled && !_scheduledFolders.isEmpty()) {
// We have things in our queue that were waiting for the connection to come back on.
startScheduledSyncSoon();
}
@ -604,19 +608,19 @@ void FolderMan::setSyncEnabled( bool enabled )
emit( folderSyncStateChange(0) );
}
void FolderMan::startScheduledSyncSoon(qint64 msMinimumDelay)
void FolderMan::startScheduledSyncSoon()
{
if (_startScheduledSyncTimer.isActive()) {
return;
}
if (_scheduleQueue.empty()) {
if (_scheduledFolders.empty()) {
return;
}
if (_currentSyncFolder) {
return;
}
qint64 msDelay = msMinimumDelay;
qint64 msDelay = 100; // 100ms minimum delay
qint64 msSinceLastSync = 0;
// Require a pause based on the duration of the last sync run.
@ -631,15 +635,6 @@ void FolderMan::startScheduledSyncSoon(qint64 msMinimumDelay)
msDelay = qMax(msDelay, pause);
}
// Punish consecutive follow-up syncs with longer delays.
if (Folder* nextFolder = _scheduleQueue.head()) {
int followUps = nextFolder->consecutiveFollowUpSyncs();
if (followUps >= 2) {
// This is okay due to the 1min maximum delay limit below.
msDelay *= qPow(followUps, 2);
}
}
// Delays beyond one minute seem too big, particularly since there
// could be things later in the queue that shouldn't be punished by a
// long delay!
@ -648,11 +643,7 @@ void FolderMan::startScheduledSyncSoon(qint64 msMinimumDelay)
// Time since the last sync run counts against the delay
msDelay = qMax(1ll, msDelay - msSinceLastSync);
// A minimum of delay here is essential as the sync will not upload
// files that were changed too recently.
msDelay = qMax(SyncEngine::minimumFileAgeForUpload, msDelay);
qDebug() << "Scheduling a sync in" << (msDelay/1000) << "seconds";
qDebug() << "Starting the next scheduled sync in" << (msDelay/1000) << "seconds";
_startScheduledSyncTimer.start(msDelay);
}
@ -673,15 +664,15 @@ void FolderMan::slotStartScheduledFolderSync()
return;
}
qDebug() << "XX slotScheduleFolderSync: folderQueue size: " << _scheduleQueue.count();
if( _scheduleQueue.isEmpty() ) {
qDebug() << "XX slotScheduleFolderSync: folderQueue size: " << _scheduledFolders.count();
if( _scheduledFolders.isEmpty() ) {
return;
}
// Find the first folder in the queue that can be synced.
Folder* f = 0;
while( !_scheduleQueue.isEmpty() ) {
f = _scheduleQueue.dequeue();
while( !_scheduledFolders.isEmpty() ) {
f = _scheduledFolders.dequeue();
Q_ASSERT(f);
if( f->canSync() ) {
@ -711,7 +702,7 @@ void FolderMan::slotEtagPollTimerTimeout()
if (_currentSyncFolder == f) {
continue;
}
if (_scheduleQueue.contains(f)) {
if (_scheduledFolders.contains(f)) {
continue;
}
if (_disabledFolders.contains(f)) {
@ -766,10 +757,54 @@ void FolderMan::slotServerVersionChanged(Account *account)
}
}
void FolderMan::slotScheduleFolderOwningFile(const QString& path)
void FolderMan::slotWatchedFileUnlocked(const QString& path)
{
if (Folder* f = folderForPath(path)) {
slotScheduleSync(f);
f->scheduleThisFolderSoon();
}
}
void FolderMan::slotScheduleFolderByTime()
{
foreach (auto& f, _folderMap) {
// Never schedule if syncing is disabled or when we're currently
// querying the server for etags
if (!f->canSync() || f->etagJob()) {
continue;
}
auto msecsSinceSync = f->msecSinceLastSync();
// Possibly it's just time for a new sync run
bool forceSyncIntervalExpired =
quint64(msecsSinceSync) > ConfigFile().forceSyncInterval();
if (forceSyncIntervalExpired) {
qDebug() << "** scheduling folder" << f->alias()
<< "because it has been" << msecsSinceSync << "ms "
<< "since the last sync";
slotScheduleSync(f);
continue;
}
// Retry a couple of times after failure
bool syncAgainAfterFail = f->consecutiveFailingSyncs() > 0 && f->consecutiveFailingSyncs() < 3;
qint64 syncAgainAfterFailDelay = 10 * 1000; // 10s for the first retry-after-fail
if (f->consecutiveFailingSyncs() > 1)
syncAgainAfterFailDelay = 60 * 1000; // 60s for each further attempt
if (syncAgainAfterFail
&& msecsSinceSync > syncAgainAfterFailDelay) {
qDebug() << "** scheduling folder" << f->alias()
<< "because the last"
<< f->consecutiveFailingSyncs() << "syncs failed, last status:"
<< f->syncResult().statusString()
<< "time since last sync:" << msecsSinceSync;
slotScheduleSync(f);
continue;
}
// Do we want to retry failing syncs or another-sync-needed runs more often?
}
}
@ -827,7 +862,6 @@ Folder* FolderMan::addFolderInternal(FolderDefinition folderDefinition, AccountS
}
// See matching disconnects in unloadFolder().
connect(folder, SIGNAL(scheduleToSync(Folder*)), SLOT(slotScheduleSync(Folder*)));
connect(folder, SIGNAL(syncStarted()), SLOT(slotFolderSyncStarted()));
connect(folder, SIGNAL(syncFinished(SyncResult)), SLOT(slotFolderSyncFinished(SyncResult)));
connect(folder, SIGNAL(syncStateChange()), SLOT(slotForwardFolderSyncStateChange()));
@ -895,7 +929,7 @@ void FolderMan::slotRemoveFolder( Folder *f )
terminateSyncProcess();
}
if (_scheduleQueue.removeAll(f) > 0) {
if (_scheduledFolders.removeAll(f) > 0) {
emit scheduleQueueChanged();
}
@ -1262,7 +1296,7 @@ void FolderMan::setIgnoreHiddenFiles(bool ignore)
QQueue<Folder*> FolderMan::scheduleQueue() const
{
return _scheduleQueue;
return _scheduledFolders;
}
Folder *FolderMan::currentSyncFolder() const

59
src/gui/folderman.h
View File

@ -37,6 +37,26 @@ class LockWatcher;
/**
* @brief The FolderMan class
* @ingroup gui
*
* The FolderMan knows about all loaded folders and is responsible for
* scheduling them when necessary.
*
* A folder is scheduled if:
* - The configured force-sync-interval has expired
* (_timeScheduler and slotScheduleFolderByTime())
*
* - A folder watcher receives a notification about a file change
* (_folderWatchers and Folder::slotWatchedPathChanged())
*
* - The folder etag on the server has changed
* (_etagPollTimer)
*
* - The locks of a monitored file are released
* (_lockWatcher and slotWatchedFileUnlocked())
*
* - There was a sync error or a follow-up sync is requested
* (_timeScheduler and slotScheduleFolderByTime()
* and Folder::slotSyncFinished())
*/
class FolderMan : public QObject
{
@ -190,6 +210,7 @@ public slots:
* Triggers a sync run once the lock on the given file is removed.
*
* Automatically detemines the folder that's responsible for the file.
* See slotWatchedFileUnlocked().
*/
void slotSyncOnceFileUnlocks(const QString& path);
@ -207,10 +228,20 @@ private slots:
void slotServerVersionChanged(Account* account);
/**
* Schedules the folder for synchronization that contains
* A file whose locks were being monitored has become unlocked.
*
* This schedules the folder for synchronization that contains
* the file with the given path.
*/
void slotScheduleFolderOwningFile(const QString& path);
void slotWatchedFileUnlocked(const QString& path);
/**
* Schedules folders whose time to sync has come.
*
* Either because a long time has passed since the last sync or
* because of previous failures.
*/
void slotScheduleFolderByTime();
private:
/** Adds a new folder, does not add it to the account settings and
@ -222,7 +253,7 @@ private:
void unloadFolder( Folder * );
/** Will start a sync after a bit of delay. */
void startScheduledSyncSoon(qint64 msMinimumDelay = 0);
void startScheduledSyncSoon();
// finds all folder configuration files
// and create the folders
@ -238,19 +269,29 @@ private:
Folder *_currentSyncFolder;
QPointer<Folder> _lastSyncFolder;
bool _syncEnabled;
QTimer _etagPollTimer;
QPointer<RequestEtagJob> _currentEtagJob; // alias of Folder running the current RequestEtagJob
/// Watching for file changes in folders
QMap<QString, FolderWatcher*> _folderWatchers;
/// Starts regular etag query jobs
QTimer _etagPollTimer;
/// The currently running etag query
QPointer<RequestEtagJob> _currentEtagJob;
/// Watches files that couldn't be synced due to locks
QScopedPointer<LockWatcher> _lockWatcher;
QScopedPointer<SocketApi> _socketApi;
/** The aliases of folders that shall be synced. */
QQueue<Folder*> _scheduleQueue;
/// Occasionally schedules folders
QTimer _timeScheduler;
/** When the timer expires one of the scheduled syncs will be started. */
/// Scheduled folders that should be synced as soon as possible
QQueue<Folder*> _scheduledFolders;
/// Picks the next scheduled folder and starts the sync
QTimer _startScheduledSyncTimer;
QScopedPointer<SocketApi> _socketApi;
bool _appRestartRequired;
static FolderMan *_instance;