Implement bookmarks backup to Files

fixes #1318

Signed-off-by: Marcel Klehr <mklehr@gmx.net>

appinfo/info.xml

@@ -43,6 +43,7 @@ Requirements:
 	<background-jobs>
 		<job>OCA\Bookmarks\BackgroundJobs\CrawlJob</job>
 		<job>OCA\Bookmarks\BackgroundJobs\FileCacheGCJob</job>
+		<job>OCA\Bookmarks\BackgroundJobs\BackupJob</job>
 	</background-jobs>
 
 	<settings>

appinfo/routes.php

@@ -134,6 +134,8 @@ return [
 		['name' => 'settings#get_view_mode', 'url' => '/settings/viewMode', 'verb' => 'GET'],
 		['name' => 'settings#set_archive_path', 'url' => '/settings/archivePath', 'verb' => 'POST'],
 		['name' => 'settings#get_archive_path', 'url' => '/settings/archivePath', 'verb' => 'GET'],
+		['name' => 'settings#set_backup_path', 'url' => '/settings/backupPath', 'verb' => 'POST'],
+		['name' => 'settings#get_backup_path', 'url' => '/settings/backupPath', 'verb' => 'GET'],
 		['name' => 'settings#get_limit', 'url' => '/settings/limit', 'verb' => 'GET'],
 
 		# public link web view

lib/BackgroundJobs/BackupJob.php

@@ -0,0 +1,95 @@
+<?php
+/*
+ * Copyright (c) 2020. The Nextcloud Bookmarks contributors.
+ *
+ * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
+ */
+
+namespace OCA\Bookmarks\BackgroundJobs;
+
+use OCA\Bookmarks\Service\BackupManager;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\TimedJob;
+use OCA\Bookmarks\Db\BookmarkMapper;
+use OCP\Http\Client\IClient;
+use OCP\Http\Client\IClientService;
+use OCP\IUser;
+use OCP\IUserManager;
+use Psr\Log\LoggerInterface;
+
+class BackupJob extends TimedJob {
+	// MAX 2880 people's bookmarks can be backupped per day
+	public const BATCH_SIZE = 10; // 10 accounts
+	public const INTERVAL = 5 * 60; // 5 minutes
+
+	/**
+	 * @var BookmarkMapper
+	 */
+	private $bookmarkMapper;
+	/**
+	 * @var IClientService
+	 */
+	private $clientService;
+	/**
+	 * @var IClient
+	 */
+	private $client;
+	/**
+	 * @var ITimeFactory
+	 */
+	private $timeFactory;
+	/**
+	 * @var IUserManager
+	 */
+	private $userManager;
+	/**
+	 * @var BackupManager
+	 */
+	private $backupManager;
+	/**
+	 * @var LoggerInterface
+	 */
+	private $logger;
+
+	public function __construct(
+		BookmarkMapper $bookmarkMapper, ITimeFactory $timeFactory, IUserManager $userManager, BackupManager $backupManager, LoggerInterface $logger
+	) {
+		parent::__construct($timeFactory);
+		$this->bookmarkMapper = $bookmarkMapper;
+
+		$this->setInterval(self::INTERVAL);
+		$this->userManager = $userManager;
+		$this->backupManager = $backupManager;
+		$this->logger = $logger;
+	}
+
+	protected function run($argument) {
+		$users = [];
+		$this->userManager->callForSeenUsers(function (IUser $user) use (&$users) {
+			$users[] = $user->getUID();
+		});
+
+		$processed = 0;
+		do {
+			$user = array_pop($users);
+			if (!$user) {
+				return;
+			}
+			try {
+				if ($this->bookmarkMapper->countBookmarksOfUser($user) === 0) {
+					continue;
+				}
+				if ($this->backupManager->backupExistsForToday($user)) {
+					continue;
+				}
+				$this->backupManager->runBackup($user);
+				$this->backupManager->cleanupOldBackups($user);
+				$processed++;
+			} catch (\Exception $e) {
+				$this->logger->warning('Bookmarks backup for user '.$user.'errored');
+				$this->logger->warning($e->getMessage());
+				continue;
+			}
+		} while ($processed < self::BATCH_SIZE);
+	}
+}

lib/Controller/SettingsController.php

@@ -171,4 +171,31 @@ class SettingsController extends ApiController {
 	public function setArchivePath(string $archivePath): JSONResponse {
 		return $this->setSetting('archive.filePath', $archivePath);
 	}
+
+	/**
+	 * get user-defined archive path
+	 *
+	 * @return JSONResponse
+	 *
+	 * @NoAdminRequired
+	 */
+	public function getBackupPath(): JSONResponse {
+		return $this->getSetting(
+			'backup.filePath',
+			'backupPath',
+			$this->l->t('Bookmarks Backups')
+		);
+	}
+
+	/**
+	 * set user-defined backup path
+	 *
+	 * @param string $backupPath
+	 * @return JSONResponse
+	 *
+	 * @NoAdminRequired
+	 */
+	public function setBackupPath(string $backupPath): JSONResponse {
+		return $this->setSetting('backup.filePath', $backupPath);
+	}
 }

lib/Service/BackupManager.php

@@ -0,0 +1,199 @@
+<?php
+/*
+ * Copyright (c) 2022. The Nextcloud Bookmarks contributors.
+ *
+ * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
+ */
+
+namespace OCA\Bookmarks\Service;
+
+use DateTime;
+use DateTimeImmutable;
+use OC\Files\Node\Folder;
+use OCA\Bookmarks\Db\FolderMapper;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\Files\IRootFolder;
+use OCP\IConfig;
+use OCP\IL10N;
+
+class BackupManager {
+
+	/**
+	 * @var IConfig
+	 */
+	private $config;
+	/**
+	 * @var string
+	 */
+	private $appName;
+	/**
+	 * @var IL10N
+	 */
+	private $l;
+	/**
+	 * @var HtmlExporter
+	 */
+	private $htmlExporter;
+	/**
+	 * @var FolderMapper
+	 */
+	private $folderMapper;
+	/**
+	 * @var ITimeFactory
+	 */
+	private $time;
+	/**
+	 * @var IRootFolder
+	 */
+	private $rootFolder;
+
+	public function __construct(string $appName, IConfig $config, IL10N $l, HtmlExporter $htmlExporter, FolderMapper $folderMapper, ITimeFactory $time, IRootFolder $rootFolder) {
+		$this->appName = $appName;
+		$this->config = $config;
+		$this->l = $l;
+		$this->htmlExporter = $htmlExporter;
+		$this->folderMapper = $folderMapper;
+		$this->time = $time;
+		$this->rootFolder = $rootFolder;
+	}
+
+	public function injectTimeFactory(ITimeFactory $time) {
+		$this->time = $time;
+	}
+
+	/**
+	 * @throws \OCP\Files\NotPermittedException
+	 * @throws \OC\User\NoUserException
+	 * @throws \Exception
+	 */
+	public function backupExistsForToday(string $userId) {
+		$path = $this->getBackupFilePathForDate($userId, $this->time->getDateTime()->getTimestamp());
+		$userFolder = $this->rootFolder->getUserFolder($userId);
+		return $userFolder->nodeExists($path);
+	}
+
+	/**
+	 * @throws \OCP\Files\NotPermittedException
+	 * @throws \OCP\AppFramework\Db\MultipleObjectsReturnedException
+	 * @throws \OCA\Bookmarks\Exception\UnauthorizedAccessError
+	 * @throws \OC\User\NoUserException
+	 * @throws \OCP\AppFramework\Db\DoesNotExistException
+	 * @throws \Exception
+	 */
+	public function runBackup($userId) {
+		$rootFolder = $this->folderMapper->findRootFolder($userId);
+		$exportedHTML = $this->htmlExporter->exportFolder($userId, $rootFolder->getId());
+
+		$userFolder = $this->rootFolder->getUserFolder($userId);
+		$folderPath = $this->getBackupFolderPath($userId);
+		if (!$userFolder->nodeExists($folderPath)) {
+			$userFolder->newFolder($folderPath);
+		}
+		$backupFilePath = $this->getBackupFilePathForDate($userId, $this->time->getDateTime()->getTimestamp());
+		$file = $userFolder->newFile($backupFilePath);
+		$file->putContent($exportedHTML);
+	}
+
+	private function getBackupFolderPath(string $userId):string {
+		return $this->config->getUserValue(
+			$userId,
+			$this->appName,
+			'backup.filePath',
+			$this->l->t('Bookmarks Backups')
+		);
+	}
+
+	/**
+	 * @throws \Exception
+	 */
+	private function getBackupFilePathForDate(string $userId, int $time) {
+		$date = DateTime::createFromFormat('U', (string)$time);
+		return $this->getBackupFolderPath($userId) . '/' . $date->format('Y-m-d') . '.html';
+	}
+
+	/**
+	 * @throws \OCP\Files\NotPermittedException
+	 * @throws \OC\User\NoUserException
+	 * @throws \OCP\Files\NotFoundException
+	 */
+	public function getBackupFolder(string $userId) : ?Folder {
+		$userFolder = $this->rootFolder->getUserFolder($userId);
+		$backupFolder = $userFolder->get($this->getBackupFolderPath($userId));
+		if (!($backupFolder instanceof Folder)) {
+			return null;
+		}
+		return $backupFolder;
+	}
+
+	/**
+	 * @throws \OCP\Files\NotPermittedException
+	 * @throws \OC\User\NoUserException
+	 * @throws \OCP\Files\NotFoundException
+	 * @throws \Exception
+	 */
+	public function cleanupOldBackups($userId) {
+		$backupFolder = $this->getBackupFolder($userId);
+		if ($backupFolder === null) {
+			return;
+		}
+		$today = DateTimeImmutable::createFromMutable($this->time->getDateTime()->setTime(0, 0));
+		$daysToKeep = [];
+		$weeksToKeep = [];
+		$monthsToKeep = [];
+		// 7 days
+		for ($i = 0; $i < 7; $i++) {
+			$daysToKeep[] = $today->sub(new \DateInterval('P'.$i.'D'));
+		}
+		// 5 weeks
+		for ($i = 1; $i < 5; $i++) {
+			$weeksToKeep[] = $today->modify('Monday this week')->sub(new \DateInterval('P'.$i.'W'));
+		}
+		// 6 months
+		for ($i = 1; $i < 6; $i++) {
+			$monthsToKeep[] = $today->modify('first day of')->sub(new \DateInterval('P'.$i.'M'));
+		}
+		$nodes = $backupFolder->getDirectoryListing();
+		foreach ($nodes as $node) {
+			if (!str_ends_with($node->getName(), '.html')) {
+				continue;
+			}
+			$date = new DateTime(basename($node->getName(), '.html'));
+			$matchingDays = count(array_filter($daysToKeep, function ($dayToKeep) use ($date) {
+				return $date->diff($dayToKeep)->days === 0;
+			}));
+			$matchingWeeks = count(array_filter($weeksToKeep, function ($weekToKeep) use ($date) {
+				return $date->diff($weekToKeep)->days === 0;
+			}));
+			$matchingMonths = count(array_filter($monthsToKeep, function ($monthToKeep) use ($date) {
+				return abs($date->diff($monthToKeep)->days) < 6;
+			}));
+			if (!$matchingDays && !$matchingWeeks && !$matchingMonths) {
+				$node->delete();
+			}
+		}
+	}
+
+	/**
+	 * @throws \OCP\Files\NotPermittedException
+	 * @throws \OC\User\NoUserException
+	 * @throws \OCP\Files\NotFoundException
+	 * @throws \Exception
+	 */
+	public function cleanupAllBackups($userId) {
+		$userFolder = $this->rootFolder->getUserFolder($userId);
+		if (!$userFolder->nodeExists($this->getBackupFolderPath($userId))) {
+			return;
+		}
+		$backupFolder = $userFolder->get($this->getBackupFolderPath($userId));
+		if (!($backupFolder instanceof Folder)) {
+			return;
+		}
+		$nodes = $backupFolder->getDirectoryListing();
+		foreach ($nodes as $node) {
+			if (!str_ends_with($node->getName(), '.html')) {
+				continue;
+			}
+			$node->delete();
+		}
+	}
+}

src/components/Settings.vue

@@ -27,6 +27,16 @@
 				@click="onChangeArchivePath">
 		</label>
 
+		<label><h3>{{ t('bookmarks', 'Backup path') }}</h3>
+			<p>{{ t('bookmarks',
+				'Enter the path of a folder in your Files where your bookmarks backups will be stored.'
+			) }}</p>
+			<input
+				:value="backupPath"
+				:readonly="true"
+				@click="onChangeBackupPath">
+		</label>
+
 		<h3>{{ t('bookmarks', 'Client apps') }}</h3>
 		<p>
 			{{
@@ -74,7 +84,13 @@ export default {
 			importing: false,
 			deleting: false,
 			addToHomeScreen: null,
-			filePicker: getFilePickerBuilder(this.t('bookmarks', 'Archive path'))
+			archivePathPicker: getFilePickerBuilder(this.t('bookmarks', 'Archive path'))
+				.allowDirectories(true)
+				.setModal(true)
+				.setType(1)// CHOOSE
+				.setMultiSelect(false)
+				.build(),
+			backupPathPicker: getFilePickerBuilder(this.t('bookmarks', 'Backup path'))
 				.allowDirectories(true)
 				.setModal(true)
 				.setType(1)// CHOOSE
@@ -94,6 +110,9 @@ export default {
 		archivePath() {
 			return this.$store.state.settings.archivePath
 		},
+		backupPath() {
+			return this.$store.state.settings.backupPath
+		},
 	},
 	mounted() {
 		window.addEventListener('beforeinstallprompt', (e) => {
@@ -123,12 +142,19 @@ export default {
 					+ encodeURIComponent(getRequestToken())
 		},
 		async onChangeArchivePath(e) {
-			const path = await this.filePicker.pick()
+			const path = await this.archivePathPicker.pick()
 			await this.$store.dispatch(actions.SET_SETTING, {
 				key: 'archivePath',
 				value: path,
 			})
 		},
+		async onChangeBackupPath(e) {
+			const path = await this.backupPathPicker.pick()
+			await this.$store.dispatch(actions.SET_SETTING, {
+				key: 'backupPath',
+				value: path,
+			})
+		},
 		clickAddToHomeScreen() {
 			if (!this.addToHomeScreen) {
 				alert(this.t('bookmarks', 'Please select "Add to home screen" in your browser menu'))

src/store/actions.js

@@ -1137,7 +1137,7 @@ export default {
 	},
 	[actions.LOAD_SETTINGS]({ commit, dispatch, state }) {
 		return Promise.all(
-			['sorting', 'viewMode', 'archivePath', 'limit'].map(key =>
+			['sorting', 'viewMode', 'archivePath', 'backupPath', 'limit'].map(key =>
 				dispatch(actions.LOAD_SETTING, key)
 			)
 		)

tests/BackupManagerTest.php

@@ -0,0 +1,95 @@
+<?php
+
+namespace OCA\Bookmarks\Tests;
+
+use DateTime;
+use OCA\Bookmarks\Exception\AlreadyExistsError;
+use OCA\Bookmarks\Exception\UrlParseError;
+use OCA\Bookmarks\Exception\UserLimitExceededError;
+use OCA\Bookmarks\Service\BackupManager;
+use OCA\Bookmarks\Service\BookmarkService;
+use OCP\AppFramework\Db\MultipleObjectsReturnedException;
+use OCP\AppFramework\Utility\ITimeFactory;
+
+class BackupManagerTest extends TestCase {
+
+	/**
+	 * @var BookmarkService
+	 */
+	private $bookmarks;
+
+	/**
+	 * @var BackupManager
+	 */
+	private $backupManager;
+
+	/**
+	 * @var string
+	 */
+	private $userId;
+	/**
+	 * @var \OC\User\Manager
+	 */
+	private $userManager;
+	/**
+	 * @var string
+	 */
+	private $user;
+
+	/**
+	 * @throws UrlParseError
+	 * @throws MultipleObjectsReturnedException
+	 * @throws \OCA\Bookmarks\Exception\UnsupportedOperation
+	 * @throws AlreadyExistsError
+	 * @throws \OCP\AppFramework\Db\DoesNotExistException
+	 * @throws UserLimitExceededError
+	 */
+	protected function setUp(): void {
+		parent::setUp();
+		$this->cleanUp();
+
+		$this->bookmarks = \OC::$server->query(BookmarkService::class);
+		$this->backupManager = \OC::$server->query(BackupManager::class);
+		$this->time = $this->createStub(ITimeFactory::class);
+		$this->backupManager->injectTimeFactory($this->time);
+
+		$this->userManager = \OC::$server->getUserManager();
+		$this->user = 'test';
+		if (!$this->userManager->userExists($this->user)) {
+			$this->userManager->createUser($this->user, 'password');
+		}
+		$this->userId = $this->userManager->get($this->user)->getUID();
+
+		$this->bookmarks->create($this->userId, 'https://en.wikipedia.org/');
+		$this->backupManager->cleanupAllBackups($this->userId);
+	}
+
+	public function testOneDay() {
+		$this->time->method('getDateTime')->willReturn(new DateTime());
+		$this->assertEquals(false, $this->backupManager->backupExistsForToday($this->userId));
+		$this->backupManager->runBackup($this->userId);
+		$this->assertEquals(true, $this->backupManager->backupExistsForToday($this->userId));
+	}
+
+	public function testSixMonths() {
+		$today = new DateTime();
+		$today->modify('first day of');
+		$sixMonths = \DateTimeImmutable::createFromMutable($today);
+		$sixMonths = $sixMonths->add(new \DateInterval('P7M'));
+		for ($i = 0; $today->diff($sixMonths)->days !== 0; $i++) {
+			$this->time->method('getDateTime')->willReturn($today);
+			$this->backupManager->runBackup($this->userId);
+			$this->backupManager->cleanupOldBackups($this->userId);
+			$today->add(new \DateInterval('P1D'));
+		}
+		$backupFolder = $this->backupManager->getBackupFolder($this->userId);
+		$nodes = $backupFolder->getDirectoryListing();
+		$backups = array_filter($nodes, function ($node) {
+			return str_ends_with($node->getName(), '.html');
+		});
+		/*var_dump(array_map(function ($node) {
+			return $node->getName();
+		}, $backups));*/
+		$this->assertEqualsWithDelta(count($backups), 7 + 5 + 5, 2);
+	}
+}