nextcloud
/
photos
mirror of https://github.com/nextcloud/photos
0
0
Fork 0
Code Issues Releases Wiki Activity

Add commands and listeners to generate location data of files: 

The location data is stored inside `oc_files_metadata`.

- `occ photos:update-1000-cities` to update the cities1000 file.
- `occ photos:map-media-to-location`to map picture coordinates to a location

- `ReverseGeoCoderService` download the necessary files and build the `KDTree`
- `UpdateReverseGeocodingFilesCommand` command to allow to manually create the needed reverse geocoding files
- `MediaLocationManager` to manager the location mappings
- `MapMediaToLocationCommand` command to manually trigger location data mapping. Useful for pre-existing pictures.
- `LocationManagerNodeEventListener` to react to node, user and share events.
- `MapMediaToLocationJob` to reduce the load in event listeners

```php
                                          ┌─────────────────────┐
                            ┌────────────►│MapMediaToLocationJob│
                            │             └─────────┬───────────┘
                            │                       │
   ┌────────────────────────┴───────┐               │
   │LocationManagerNodeEventListener├──┐            ▼
   └────────────────────────────────┘  │  ┌────────────────────┐     ┌──────────────┐
                                       ├─►│MediaLocationManager├────►│LocationMapper│
   ┌─────────────────────────┐         │  └─────────┬──────────┘     └──────────────┘
   │MapMediaToLocationCommand├─────────┘            │
   └─────────────────────────┘                      │
                                                    ▼
   ┌──────────────────────────────────┐   ┌──────────────────────┐
   │UpdateReverseGeocodingFilesCommand├──►│ReverseGeoCoderService│
   └──────────────────────────────────┘   └──────────────────────┘
```

Signed-off-by: Louis Chemineau <louis@chmn.me>
This commit is contained in:
Louis Chemineau 2023-02-22 14:58:09 +01:00
parent 79217e6105
commit d94f30cc09
20 changed files with 1099 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/cypress.yml vendored
View File

@ -19,6 +19,9 @@ jobs:
- name: Checkout app
uses: actions/checkout@v3
- name: Install server dependencies
run: composer install
- name: Read package.json node and npm engines version
uses: skjnldsv/read-package-engines-version-actions@v2.0
id: versions
@ -35,7 +38,7 @@ jobs:
- name: Set up npm ${{ steps.versions.outputs.npmVersion }}
run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}"
- name: Install dependencies & build app
- name: Install node dependencies & build app
run: |
npm ci
TESTING=true npm run build --if-present

2
.php-cs-fixer.dist.php
View File

@ -2,7 +2,7 @@
declare(strict_types=1);
require_once './vendor/autoload.php';
require_once __DIR__ . '/vendor/autoload.php';
use Nextcloud\CodingStandard\Config;

21
appinfo/info.xml
View File

@ -1,13 +1,13 @@
<?xml version="1.0"?>
<info xmlns:xsi= "http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="https://apps.nextcloud.com/schema/apps/info.xsd">
<info xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="https://apps.nextcloud.com/schema/apps/info.xsd">
<id>photos</id>
<name>Photos</name>
<summary>Your memories under your control</summary>
<description>Your memories under your control</description>
<version>2.2.0</version>
<licence>agpl</licence>
<author mail="skjnldsv@protonmail.com">John Molakvoæ</author>
<author mail="skjnldsv@protonmail.com">John Molakvoæ</author>
<namespace>Photos</namespace>
<category>multimedia</category>
<types>
@ -15,8 +15,8 @@
<authentication />
</types>
<website>https://github.com/nextcloud/photos</website>
<bugs>https://github.com/nextcloud/photos/issues</bugs>
<website>https://github.com/nextcloud/photos</website>
<bugs>https://github.com/nextcloud/photos/issues</bugs>
<repository>https://github.com/nextcloud/photos.git</repository>
<default_enable />
<dependencies>
@ -30,6 +30,11 @@
</navigation>
</navigations>
<commands>
<command>OCA\Photos\Command\UpdateReverseGeocodingFilesCommand</command>
<command>OCA\Photos\Command\MapMediaToLocationCommand</command>
</commands>
<sabre>
<collections>
<collection>OCA\Photos\Sabre\RootCollection</collection>
@ -39,4 +44,8 @@
<plugin>OCA\Photos\Sabre\Album\PropFindPlugin</plugin>
</plugins>
</sabre>
</info>
<background-jobs>
<job>OCA\Photos\Jobs\AutomaticLocationMapperJob</job>
</background-jobs>
</info>

3
composer.json
View File

@ -21,5 +21,8 @@
"vimeo/psalm": "^4.22",
"sabre/dav": "^4.2.1",
"nextcloud/ocp": "dev-master"
},
"require": {
"hexogen/kdtree": "^0.2.5"
}
}

66
composer.lock generated
View File

@ -4,8 +4,70 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "602bc2404448c321b5bbb50160f12007",
"packages": [],
"content-hash": "88bed2a916ac8f06153e69bb691c154c",
"packages": [
{
"name": "hexogen/kdtree",
"version": "v0.2.5",
"source": {
"type": "git",
"url": "https://github.com/hexogen/kdtree.git",
"reference": "f739186638445990463762d467e07a8262228daa"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/hexogen/kdtree/zipball/f739186638445990463762d467e07a8262228daa",
"reference": "f739186638445990463762d467e07a8262228daa",
"shasum": ""
},
"require": {
"php": "^7.1|^8.0"
},
"require-dev": {
"league/csv": "^9.7.0",
"mockery/mockery": "dev-master",
"phpunit/phpunit": "^9.0",
"squizlabs/php_codesniffer": "^3.5.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.0-dev"
}
},
"autoload": {
"psr-4": {
"Hexogen\\KDTree\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Volodymyr Basarab",
"email": "volodymyrbas@gmail.com",
"homepage": "https://github.com/hexogen",
"role": "Developer"
}
],
"description": "file system KDTree index",
"homepage": "https://github.com/hexogen/kdtree",
"keywords": [
"algorithms",
"data structures",
"hexogen",
"kdtree",
"search"
],
"support": {
"issues": "https://github.com/hexogen/kdtree/issues",
"source": "https://github.com/hexogen/kdtree/tree/v0.2.5"
},
"time": "2022-11-21T13:19:19+00:00"
}
],
"packages-dev": [
{
"name": "amphp/amp",

63
lib/Album/AlbumFile.php
View File

@ -23,19 +23,11 @@ declare(strict_types=1);
namespace OCA\Photos\Album;
use OC\Metadata\FileMetadata;
use OCA\Photos\DB\PhotosFile;
class AlbumFile {
private int $fileId;
private string $name;
private string $mimeType;
private int $size;
private int $mtime;
private string $etag;
class AlbumFile extends PhotosFile {
private int $added;
private string $owner;
/** @var array<string, FileMetadata> */
private array $metaData = [];
public function __construct(
int $fileId,
@ -47,52 +39,19 @@ class AlbumFile {
int $added,
string $owner
) {
$this->fileId = $fileId;
$this->name = $name;
$this->mimeType = $mimeType;
$this->size = $size;
$this->mtime = $mtime;
$this->etag = $etag;
parent::__construct(
$fileId,
$name,
$mimeType,
$size,
$mtime,
$etag
);
$this->added = $added;
$this->owner = $owner;
}
public function getFileId(): int {
return $this->fileId;
}
public function getName(): string {
return $this->name;
}
public function getMimeType(): string {
return $this->mimeType;
}
public function getSize(): int {
return $this->size;
}
public function getMTime(): int {
return $this->mtime;
}
public function getEtag(): string {
return $this->etag;
}
public function setMetadata(string $key, FileMetadata $value): void {
$this->metaData[$key] = $value;
}
public function hasMetadata(string $key): bool {
return isset($this->metaData[$key]);
}
public function getMetadata(string $key): FileMetadata {
return $this->metaData[$key];
}
public function getAdded(): int {
return $this->added;
}

7
lib/AppInfo/Application.php
View File

@ -32,6 +32,7 @@ use OCA\Photos\Listener\NodeDeletedListener;
use OCA\Photos\Listener\TagListener;
use OCA\Photos\Listener\GroupUserRemovedListener;
use OCA\Photos\Listener\GroupDeletedListener;
use OCA\Photos\Listener\LocationManagerEventListener;
use OCP\AppFramework\App;
use OCP\AppFramework\Bootstrap\IBootContext;
use OCP\AppFramework\Bootstrap\IBootstrap;
@ -40,6 +41,9 @@ use OCP\Files\Events\Node\NodeDeletedEvent;
use OCP\SystemTag\MapperEvent;
use OCP\Group\Events\UserRemovedEvent;
use OCP\Group\Events\GroupDeletedEvent;
use OCP\Files\Events\Node\NodeWrittenEvent;
require_once __DIR__ . '/../../vendor/autoload.php';
class Application extends App implements IBootstrap {
public const APP_ID = 'photos';
@ -78,6 +82,9 @@ class Application extends App implements IBootstrap {
$context->registerEventListener(GroupDeletedEvent::class, GroupDeletedListener::class);
// Priority of -1 to be triggered after event listeners populating metadata.
$context->registerEventListener(NodeWrittenEvent::class, LocationManagerEventListener::class, -1);
$context->registerEventListener(SabrePluginAuthInitEvent::class, SabrePluginAuthInitListener::class);
$context->registerEventListener(MapperEvent::EVENT_ASSIGN, TagListener::class);

116
lib/Command/MapMediaToLocationCommand.php Normal file
View File

@ -0,0 +1,116 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Louis Chemineau <louis@chmn.me>
*
* @author Louis Chemineau <louis@chmn.me>
*
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Photos\Command;
use OCP\IConfig;
use OCP\IUserManager;
use OCP\Files\IRootFolder;
use OCP\Files\Folder;
use OCA\Photos\Service\MediaLocationManager;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class MapMediaToLocationCommand extends Command {
public function __construct(
private IRootFolder $rootFolder,
private MediaLocationManager $mediaLocationManager,
private IConfig $config,
private IUserManager $userManager,
) {
parent::__construct();
}
/**
* Configure the command
*/
protected function configure(): void {
$this->setName('photos:map-media-to-location')
->setDescription('Reverse geocode media coordinates.')
->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Limit the mapping to a user.', null);
}
/**
* Execute the command
*/
protected function execute(InputInterface $input, OutputInterface $output): int {
if (!$this->config->getSystemValueBool('enable_file_metadata', true)) {
throw new \Exception('File metadata is not enabled.');
}
$userId = $input->getOption('user');
if ($userId === null) {
$this->scanForAllUsers($output);
} else {
$this->scanFilesForUser($userId, $output);
}
return 0;
}
private function scanForAllUsers(OutputInterface $output): void {
$users = $this->userManager->search('');
$output->writeln("Scanning all users:");
foreach ($users as $user) {
$this->scanFilesForUser($user->getUID(), $output);
}
}
private function scanFilesForUser(string $userId, OutputInterface $output): void {
$userFolder = $this->rootFolder->getUserFolder($userId);
$output->write(" - Scanning files for $userId");
$startTime = time();
$count = $this->scanFolder($userFolder);
$timeElapse = time() - $startTime;
$output->writeln(" - $count files, $timeElapse sec");
}
private function scanFolder(Folder $folder): int {
$count = 0;
// Do not scan share and other moveable mounts.
if ($folder->getMountPoint() instanceof \OC\Files\Mount\MoveableMount) {
return $count;
}
foreach ($folder->getDirectoryListing() as $node) {
if ($node instanceof Folder) {
$count += $this->scanFolder($node);
continue;
}
if (!str_starts_with($node->getMimeType(), 'image')) {
continue;
}
$this->mediaLocationManager->setLocationForFile($node->getId());
$count++;
}
return $count;
}
}

61
lib/Command/UpdateReverseGeocodingFilesCommand.php Normal file
View File

@ -0,0 +1,61 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Louis Chemineau <louis@chmn.me>
*
* @author Louis Chemineau <louis@chmn.me>
*
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Photos\Command;
use OCA\Photos\Service\ReverseGeoCoderService;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class UpdateReverseGeocodingFilesCommand extends Command {
public function __construct(
private ReverseGeoCoderService $rgcService,
) {
parent::__construct();
}
/**
* Configure the command
*/
protected function configure(): void {
$this->setName('photos:update-1000-cities')
->setDescription('Update the list of 1000 and more inhabitant cities');
}
/**
* Execute the command
*/
protected function execute(InputInterface $input, OutputInterface $output): int {
try {
$this->rgcService->buildKDTree(true);
} catch (\Exception $ex) {
$output->writeln('<error>Failed to update reverse geocoding files</error>');
$output->writeln($ex->getMessage());
return 1;
}
return 0;
}
}

53
lib/DB/Location/LocationFile.php Normal file
View File

@ -0,0 +1,53 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Louis Chemineau <louis@chmn.me>
*
* @author Louis Chemineau <louis@chmn.me>
*
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Photos\DB\Location;
use OCA\Photos\DB\PhotosFile;
class LocationFile extends PhotosFile {
public function __construct(
int $fileId,
string $name,
string $mimeType,
int $size,
int $mtime,
string $etag,
private string $location,
) {
parent::__construct(
$fileId,
$name,
$mimeType,
$size,
$mtime,
$etag,
);
}
public function getLocation(): string {
return $this->location;
}
}

42
lib/DB/Location/LocationInfo.php Normal file
View File

@ -0,0 +1,42 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Louis Chemineau <louis@chmn.me>
*
* @author Louis Chemineau <louis@chmn.me>
*
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Photos\DB\Location;
class LocationInfo {
public function __construct(
private string $userId,
private string $location
) {
}
public function getUserId(): string {
return $this->userId;
}
public function getLocation(): string {
return $this->location;
}
}

127
lib/DB/Location/LocationMapper.php Normal file
View File

@ -0,0 +1,127 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Louis Chemineau <louis@chmn.me>
*
* @author Louis Chemineau <louis@chmn.me>
*
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Photos\DB\Location;
use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\IMimeTypeLoader;
use OCP\Files\IRootFolder;
use OCP\IDBConnection;
class LocationMapper {
public const METADATA_TYPE = 'photos_location';
public function __construct(
private IDBConnection $connection,
private IMimeTypeLoader $mimeTypeLoader,
private IRootFolder $rootFolder,
) {
}
/** @return LocationInfo[] */
public function findLocationsForUser(string $userId): array {
$mountId = $this->rootFolder
->getUserFolder($userId)
->getMountPoint()
->getMountId();
$mimepart = $this->mimeTypeLoader->getId('image');
$qb = $this->connection->getQueryBuilder();
$rows = $qb->selectDistinct('meta.metadata')
->from('mounts', 'mount')
->join('mount', 'filecache', 'file', $qb->expr()->eq('file.storage', 'mount.storage_id', IQueryBuilder::PARAM_INT))
->join('file', 'file_metadata', 'meta', $qb->expr()->eq('file.fileid', 'meta.id', IQueryBuilder::PARAM_INT))
->where($qb->expr()->eq('mount.id', $qb->createNamedParameter($mountId), IQueryBuilder::PARAM_INT))
->andWhere($qb->expr()->eq('file.mimepart', $qb->createNamedParameter($mimepart, IQueryBuilder::PARAM_INT)))
->andWhere($qb->expr()->eq('meta.group_name', $qb->createNamedParameter(self::METADATA_TYPE)))
->executeQuery()
->fetchAll();
return array_map(fn ($row) => new LocationInfo($userId, $row['metadata']), $rows);
}
/** @return LocationFile[] */
public function findFilesForUserAndLocation(string $userId, string $location) {
$mountId = $this->rootFolder
->getUserFolder($userId)
->getMountPoint()
->getMountId();
$mimepart = $this->mimeTypeLoader->getId('image');
$qb = $this->connection->getQueryBuilder();
$rows = $qb->select('file.fileid', 'file.name', 'file.mimetype', 'file.size', 'file.mtime', 'file.etag', 'meta.metadata')
->from('mounts', 'mount')
->join('mount', 'filecache', 'file', $qb->expr()->eq('file.storage', 'mount.storage_id', IQueryBuilder::PARAM_INT))
->join('file', 'file_metadata', 'meta', $qb->expr()->eq('file.fileid', 'meta.id', IQueryBuilder::PARAM_INT))
->where($qb->expr()->eq('mount.id', $qb->createNamedParameter($mountId), IQueryBuilder::PARAM_INT))
->andWhere($qb->expr()->eq('file.mimepart', $qb->createNamedParameter($mimepart, IQueryBuilder::PARAM_INT)))
->andWhere($qb->expr()->eq('meta.group_name', $qb->createNamedParameter(self::METADATA_TYPE)))
->andWhere($qb->expr()->eq('meta.metadata', $qb->createNamedParameter($location)))
->executeQuery()
->fetchAll();
return array_map(
fn ($row) => new LocationFile(
(int)$row['fileid'],
$row['name'],
$this->mimeTypeLoader->getMimetypeById($row['mimetype']),
(int)$row['size'],
(int)$row['mtime'],
$row['etag'],
$row['metadata']
),
$rows,
);
}
public function setLocationForFile(string $location, int $fileId): void {
try {
$query = $this->connection->getQueryBuilder();
$query->insert('file_metadata')
->values([
"id" => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT),
"group_name" => $query->createNamedParameter(self::METADATA_TYPE),
"metadata" => $query->createNamedParameter($location),
])
->executeStatement();
} catch (\Exception $ex) {
if ($ex->getPrevious() instanceof UniqueConstraintViolationException) {
$this->updateLocationForFile($location, $fileId);
}
}
}
public function updateLocationForFile(string $location, int $fileId): void {
$query = $this->connection->getQueryBuilder();
$query->update('file_metadata')
->set("metadata", $query->createNamedParameter($location))
->where($query->expr()->eq('id', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('group_name', $query->createNamedParameter(self::METADATA_TYPE)))
->executeStatement();
}
}

79
lib/DB/PhotosFile.php Normal file
View File

@ -0,0 +1,79 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Louis Chemineau <louis@chmn.me>
*
* @author Louis Chemineau <louis@chmn.me>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Photos\DB;
use OC\Metadata\FileMetadata;
class PhotosFile {
/** @var array<string, FileMetadata> */
private array $metaData = [];
public function __construct(
private int $fileId,
private string $name,
private string $mimeType,
private int $size,
private int $mtime,
private string $etag,
) {
}
public function getFileId(): int {
return $this->fileId;
}
public function getName(): string {
return $this->name;
}
public function getMimeType(): string {
return $this->mimeType;
}
public function getSize(): int {
return $this->size;
}
public function getMTime(): int {
return $this->mtime;
}
public function getEtag(): string {
return $this->etag;
}
public function setMetadata(string $key, FileMetadata $value): void {
$this->metaData[$key] = $value;
}
public function hasMetadata(string $key): bool {
return isset($this->metaData[$key]);
}
public function getMetadata(string $key): FileMetadata {
return $this->metaData[$key];
}
}

114
lib/Jobs/AutomaticLocationMapperJob.php Normal file
View File

@ -0,0 +1,114 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Louis Chemineau <louis@chmn.me>
*
* @author Louis Chemineau <louis@chmn.me>
*
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Photos\Jobs;
use OCA\Photos\AppInfo\Application;
use OCA\Photos\Service\MediaLocationManager;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\TimedJob;
use OCP\Files\Folder;
use OCP\Files\IRootFolder;
use OCP\IConfig;
use OCP\IUserManager;
class AutomaticLocationMapperJob extends TimedJob {
public function __construct(
ITimeFactory $time,
private IConfig $config,
private IRootFolder $rootFolder,
private IUserManager $userManager,
private MediaLocationManager $mediaLocationManager,
) {
parent::__construct($time);
$this->mediaLocationManager = $mediaLocationManager;
$this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_INSENSITIVE);
$this->setInterval(24 * 3600);
}
protected function run($argument) {
$locationMappingDone = $this->config->getAppValue(Application::APP_ID, 'lastLocationMappingDone', 'false');
if ($locationMappingDone === 'true') {
return;
}
$users = $this->userManager->search('');
$lastMappedUser = $this->config->getAppValue(Application::APP_ID, 'lastLocationMappedUser', '');
if ($lastMappedUser === '') {
$lastMappedUser = $users[array_key_first($users)]->getUID();
}
$startTime = null;
foreach ($users as $user) {
if ($startTime === null) {
// Skip all user before lastMappedUser.
if ($lastMappedUser !== $user->getUID()) {
continue;
}
$startTime = time();
}
// Stop if execution time is more than one hour.
if (time() - $startTime > 60 * 60) {
return;
}
$this->scanFilesForUser($user->getUID());
$this->config->setAppValue(Application::APP_ID, 'lastLocationMappedUser', $user->getUID());
}
$this->config->setAppValue(Application::APP_ID, 'lastLocationMappingDone', 'true');
}
private function scanFilesForUser(string $userId): void {
$userFolder = $this->rootFolder->getUserFolder($userId);
$this->scanFolder($userFolder);
}
private function scanFolder(Folder $folder): void {
// Do not scan share and other moveable mounts.
if ($folder->getMountPoint() instanceof \OC\Files\Mount\MoveableMount) {
return;
}
foreach ($folder->getDirectoryListing() as $node) {
if ($node instanceof Folder) {
$this->scanFolder($node);
continue;
}
if (!str_starts_with($node->getMimeType(), 'image')) {
continue;
}
$this->mediaLocationManager->setLocationForFile($node->getId());
}
}
}

48
lib/Jobs/MapMediaToLocationJob.php Normal file
View File

@ -0,0 +1,48 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Louis Chemineau <louis@chmn.me>
*
* @author Louis Chemineau <louis@chmn.me>
*
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Photos\Jobs;
use OCA\Photos\Service\MediaLocationManager;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\QueuedJob;
class MapMediaToLocationJob extends QueuedJob {
private MediaLocationManager $mediaLocationManager;
public function __construct(
ITimeFactory $time,
MediaLocationManager $mediaLocationManager
) {
parent::__construct($time);
$this->mediaLocationManager = $mediaLocationManager;
}
protected function run($argument) {
[$fileId] = $argument;
$this->mediaLocationManager->setLocationForFile($fileId);
}
}

71
lib/Listener/LocationManagerEventListener.php Normal file
View File

@ -0,0 +1,71 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Louis Chemineau <louis@chmn.me>
*
* @author Louis Chemineau <louis@chmn.me>
*
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Photos\Listener;
use OCA\Photos\Jobs\MapMediaToLocationJob;
use OCA\Photos\Service\MediaLocationManager;
use OCP\BackgroundJob\IJobList;
use OCP\IConfig;
use OCP\EventDispatcher\Event;
use OCP\EventDispatcher\IEventListener;
use OCP\Files\Events\Node\NodeWrittenEvent;
/**
* Listener to create, update or remove location info from the database.
*/
class LocationManagerEventListener implements IEventListener {
public function __construct(
private MediaLocationManager $mediaLocationManager,
private IConfig $config,
private IJobList $jobList,
) {
}
public function handle(Event $event): void {
if (!$this->config->getSystemValueBool('enable_file_metadata', true)) {
return;
}
if ($event instanceof NodeWrittenEvent) {
if (!$this->isCorrectPath($event->getNode()->getPath())) {
return;
}
if (!str_starts_with($event->getNode()->getMimeType(), 'image')) {
return;
}
$fileId = $event->getNode()->getId();
$this->jobList->add(MapMediaToLocationJob::class, [$fileId]);
}
}
private function isCorrectPath(string $path): bool {
// TODO make this more dynamic, we have the same issue in other places
return !str_starts_with($path, 'appdata_') && !str_starts_with($path, 'files_versions/');
}
}

76
lib/Service/MediaLocationManager.php Normal file
View File

@ -0,0 +1,76 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Louis Chemineau <louis@chmn.me>
*
* @author Louis Chemineau <louis@chmn.me>
*
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Photos\Service;
use OC\Metadata\IMetadataManager;
use OCA\Photos\DB\Location\LocationMapper;
class MediaLocationManager {
public function __construct(
private IMetadataManager $metadataManager,
private ReverseGeoCoderService $rgcService,
private LocationMapper $locationMapper,
) {
}
public function setLocationForFile(int $fileId): void {
$location = $this->getLocationForFile($fileId);
if ($location === null) {
return;
}
$this->locationMapper->setLocationForFile($location, $fileId);
}
public function updateLocationForFile(int $fileId): void {
$location = $this->getLocationForFile($fileId);
if ($location === null) {
return;
}
$this->locationMapper->updateLocationForFile($location, $fileId);
}
private function getLocationForFile(int $fileId): ?string {
$gpsMetadata = $this->metadataManager->fetchMetadataFor('gps', [$fileId])[$fileId];
$metadata = $gpsMetadata->getMetadata();
if (count($metadata) === 0) {
return null;
}
$latitude = $metadata['latitude'];
$longitude = $metadata['longitude'];
if ($latitude === null || $longitude === null) {
return null;
}
return $this->rgcService->getLocationForCoordinates($latitude, $longitude);
}
}

168
lib/Service/ReverseGeoCoderService.php Normal file
View File

@ -0,0 +1,168 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2022 Louis Chemineau <louis@chmn.me>
*
* @author Louis Chemineau <louis@chmn.me>
*
* @license AGPL-3.0-or-later
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Photos\Service;
use OCP\Files\IAppData;
use OCP\Files\SimpleFS\ISimpleFolder;
use OCP\Files\NotFoundException;
use OCP\Http\Client\IClientService;
use Hexogen\KDTree\FSTreePersister;
use Hexogen\KDTree\FSKDTree;
use Hexogen\KDTree\KDTree;
use Hexogen\KDTree\Item;
use Hexogen\KDTree\ItemList;
use Hexogen\KDTree\ItemFactory;
use Hexogen\KDTree\NearestSearch;
use Hexogen\KDTree\Point;
class ReverseGeoCoderService {
private ISimpleFolder $geoNameFolder;
private ?NearestSearch $fsSearcher = null;
/** @var array<int, string> */
private ?array $citiesMapping = null;
public function __construct(
IAppData $appData,
private IClientService $clientService,
) {
try {
$this->geoNameFolder = $appData->getFolder("geonames");
} catch (NotFoundException $ex) {
$this->geoNameFolder = $appData->newFolder("geonames");
}
}
public function getLocationForCoordinates(float $latitude, float $longitude): string {
$this->loadKdTree();
$result = $this->fsSearcher->search(new Point([$latitude, $longitude]), 1);
return $this->getLocationNameForLocationId($result[0]->getId());
}
private function getLocationNameForLocationId(int $locationId): string {
if ($this->citiesMapping === null) {
$this->downloadCities1000();
$cities1000 = $this->loadCities1000();
$this->citiesMapping = [];
foreach ($cities1000 as $city) {
$this->citiesMapping[$city['id']] = $city['name'];
}
}
return $this->citiesMapping[$locationId];
}
private function downloadCities1000(bool $force = false): void {
if ($this->geoNameFolder->fileExists('cities1000.csv') && !$force) {
return;
}
// Download zip file to a tmp file.
$response = $this->clientService->newClient()->get("https://download.geonames.org/export/dump/cities1000.zip");
$tmpFile = tmpfile();
$cities1000ZipTmpFileName = stream_get_meta_data($tmpFile)['uri'];
fclose($tmpFile);
file_put_contents($cities1000ZipTmpFileName, $response->getBody());
// Unzip the txt file into a stream.
$zip = new \ZipArchive;
$res = $zip->open($cities1000ZipTmpFileName);
if ($res !== true) {
throw new \Exception("Fail to unzip location file: $res", $res);
}
$cities1000TxtSteam = $zip->getStream('cities1000.txt');
// Dump the txt file info into a smaller csv file.
$destinationStream = $this->geoNameFolder->newFile('cities1000.csv')->write();
while (($fields = fgetcsv($cities1000TxtSteam, 0, " ")) !== false) {
$result = fputcsv(
$destinationStream,
[
'id' => (int)$fields[0],
'name' => $fields[1],
'latitude' => (float)$fields[4],
'longitude' => (float)$fields[5],
]
);
if ($result === false) {
throw new \Exception('Failed to write csv line to tmp stream');
}
}
$zip->close();
}
private function loadCities1000(): array {
$csvStream = $this->geoNameFolder->getFile('cities1000.csv')->read();
$cities = [];
while (($fields = fgetcsv($csvStream)) !== false) {
$cities[] = [
'id' => (int)$fields[0],
'name' => $fields[1],
'latitude' => (float)$fields[2],
'longitude' => (float)$fields[3],
];
}
return $cities;
}
public function buildKDTree($force = false): void {
if ($this->geoNameFolder->fileExists('cities1000.bin') && !$force) {
return;
}
$this->downloadCities1000($force);
$cities1000 = $this->loadCities1000();
$itemList = new ItemList(2);
foreach ($cities1000 as $city) {
$itemList->addItem(new Item($city['id'], [$city['latitude'], $city['longitude']]));
}
$tree = new KDTree($itemList);
// Persiste KDTree in app data.
$persister = new FSTreePersister('/');
$kdTreeTmpFileName = tempnam(sys_get_temp_dir(), "nextcloud_photos_");
$persister->convert($tree, $kdTreeTmpFileName);
$kdTreeString = file_get_contents($kdTreeTmpFileName);
$this->geoNameFolder->newFile('cities1000.bin', $kdTreeString);
}
private function loadKdTree(): void {
if ($this->fsSearcher !== null) {
return;
}
$this->buildKDTree();
$kdTreeFileContent = $this->geoNameFolder->getFile("cities1000.bin")->getContent();
$kdTreeTmpFileName = tempnam(sys_get_temp_dir(), "nextcloud_photos_");
file_put_contents($kdTreeTmpFileName, $kdTreeFileContent);
$fsTree = new FSKDTree($kdTreeTmpFileName, new ItemFactory());
$this->fsSearcher = new NearestSearch($fsTree);
}
}

2
package.json
View File

@ -1,7 +1,7 @@
{
"name": "photos",
"description": "Your memories under your control",
"version": "2.2.0",
"version": "2.2.1",
"author": "John Molakvoæ <skjnldsv@protonmail.com>",
"contributors": [
"John Molakvoæ <skjnldsv@protonmail.com>"

53
tests/stub.phpstub
View File

@ -118,6 +118,7 @@ namespace Symfony\Component\Console\Question {
namespace Symfony\Component\Console\Output {
class OutputInterface {
public const VERBOSITY_VERBOSE = 1;
public function write($messages, $newline = false, $options = 0);
public function writeln(string $text, int $flat = 0) {}
}
}
@ -269,10 +270,10 @@ namespace OC\Files\Mount {
protected $class;
protected $storageId;
protected $rootId = null;
/** @var int|null */
protected $mountId;
/**
* @param string|\OCP\Files\Storage\IStorage $storage
* @param string $mountpoint
@ -285,7 +286,7 @@ namespace OC\Files\Mount {
public function __construct($storage, $mountpoint, $arguments = null, $loader = null, $mountOptions = null, $mountId = null) {
throw new \Exception('stub');
}
/**
* get complete path to the mount point, relative to data/
*
@ -294,7 +295,7 @@ namespace OC\Files\Mount {
public function getMountPoint() {
throw new \Exception('stub');
}
/**
* Sets the mount point path, relative to data/
*
@ -303,28 +304,28 @@ namespace OC\Files\Mount {
public function setMountPoint($mountPoint) {
throw new \Exception('stub');
}
/**
* @return \OCP\Files\Storage\IStorage
*/
public function getStorage() {
throw new \Exception('stub');
}
/**
* @return string
*/
public function getStorageId() {
throw new \Exception('stub');
}
/**
* @return int
*/
public function getNumericStorageId() {
throw new \Exception('stub');
}
/**
* @param string $path
* @return string
@ -332,14 +333,14 @@ namespace OC\Files\Mount {
public function getInternalPath($path) {
throw new \Exception('stub');
}
/**
* @param callable $wrapper
*/
public function wrapStorage($wrapper) {
throw new \Exception('stub');
}
/**
* Get a mount option
*
@ -350,7 +351,7 @@ namespace OC\Files\Mount {
public function getOption($name, $default) {
throw new \Exception('stub');
}
/**
* Get all options for the mount
*
@ -359,18 +360,18 @@ namespace OC\Files\Mount {
public function getOptions() {
throw new \Exception('stub');
}
/**
* @return int
*/
public function getStorageRootId() {
throw new \Exception('stub');
}
public function getMountId() {
throw new \Exception('stub');
}
public function getMountType() {
throw new \Exception('stub');
}
@ -656,7 +657,7 @@ use OCP\DB\Types;
/**
* @method string getGroupName()
* @method void setGroupName(string $groupName)
* @method string getMetadata()
* @method array getMetadata()
* @method void setMetadata(array $metadata)
* @see \OC\Core\Migrations\Version240000Date20220404230027
*/
@ -686,3 +687,25 @@ namespace OCA\DAV\Upload {
namespace Doctrine\DBAL\Exception {
class UniqueConstraintViolationException extends \Exception {}
}
namespace OC\Files\Mount;
/**
* Defines the mount point to be (re)moved by the user
*/
interface MoveableMount {
/**
* Move the mount point to $target
*
* @param string $target the target mount point
* @return bool
*/
public function moveMount($target);
/**
* Remove the mount points
*
* @return bool
*/
public function removeMount();
}