mirror of https://github.com/nextcloud/server
store unencrypted size in the unencrypted_size column
Signed-off-by: Robin Appelman <robin@icewind.nl>
This commit is contained in:
parent
5e375d9092
commit
1374cbee3e
|
@ -37,6 +37,7 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>
|
||||
*
|
||||
*/
|
||||
|
||||
namespace OC\Files\Cache;
|
||||
|
||||
use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
|
||||
|
@ -188,6 +189,7 @@ class Cache implements ICache {
|
|||
$data['fileid'] = (int)$data['fileid'];
|
||||
$data['parent'] = (int)$data['parent'];
|
||||
$data['size'] = 0 + $data['size'];
|
||||
$data['unencrypted_size'] = 0 + ($data['unencrypted_size'] ?? 0);
|
||||
$data['mtime'] = (int)$data['mtime'];
|
||||
$data['storage_mtime'] = (int)$data['storage_mtime'];
|
||||
$data['encryptedVersion'] = (int)$data['encrypted'];
|
||||
|
@ -428,7 +430,7 @@ class Cache implements ICache {
|
|||
protected function normalizeData(array $data): array {
|
||||
$fields = [
|
||||
'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted',
|
||||
'etag', 'permissions', 'checksum', 'storage'];
|
||||
'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size'];
|
||||
$extensionFields = ['metadata_etag', 'creation_time', 'upload_time'];
|
||||
|
||||
$doNotCopyStorageMTime = false;
|
||||
|
@ -873,18 +875,32 @@ class Cache implements ICache {
|
|||
$id = $entry['fileid'];
|
||||
|
||||
$query = $this->getQueryBuilder();
|
||||
$query->selectAlias($query->func()->sum('size'), 'f1')
|
||||
->selectAlias($query->func()->min('size'), 'f2')
|
||||
$query->select('size', 'unencrypted_size')
|
||||
->from('filecache')
|
||||
->whereStorageId($this->getNumericStorageId())
|
||||
->whereParent($id);
|
||||
|
||||
$result = $query->execute();
|
||||
$row = $result->fetch();
|
||||
$rows = $result->fetchAll();
|
||||
$result->closeCursor();
|
||||
|
||||
if ($row) {
|
||||
[$sum, $min] = array_values($row);
|
||||
if ($rows) {
|
||||
$sizes = array_map(function (array $row) {
|
||||
return (int)$row['size'];
|
||||
}, $rows);
|
||||
$unencryptedOnlySizes = array_map(function (array $row) {
|
||||
return (int)$row['unencrypted_size'];
|
||||
}, $rows);
|
||||
$unencryptedSizes = array_map(function (array $row) {
|
||||
return (int)(($row['unencrypted_size'] > 0) ? $row['unencrypted_size']: $row['size']);
|
||||
}, $rows);
|
||||
|
||||
$sum = array_sum($sizes);
|
||||
$min = min($sizes);
|
||||
|
||||
$unencryptedSum = array_sum($unencryptedSizes);
|
||||
$unencryptedMin = min($unencryptedSizes);
|
||||
$unencryptedMax = max($unencryptedOnlySizes);
|
||||
|
||||
$sum = 0 + $sum;
|
||||
$min = 0 + $min;
|
||||
if ($min === -1) {
|
||||
|
@ -892,8 +908,23 @@ class Cache implements ICache {
|
|||
} else {
|
||||
$totalSize = $sum;
|
||||
}
|
||||
if ($unencryptedMin === -1 || $min === -1) {
|
||||
$unencryptedTotal = $unencryptedMin;
|
||||
} else {
|
||||
$unencryptedTotal = $unencryptedSum;
|
||||
}
|
||||
if ($entry['size'] !== $totalSize) {
|
||||
$this->update($id, ['size' => $totalSize]);
|
||||
// only set unencrypted size for a folder if any child entries have it set
|
||||
if ($unencryptedMax > 0) {
|
||||
$this->update($id, [
|
||||
'size' => $totalSize,
|
||||
'unencrypted_size' => $unencryptedTotal,
|
||||
]);
|
||||
} else {
|
||||
$this->update($id, [
|
||||
'size' => $totalSize,
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -132,4 +132,12 @@ class CacheEntry implements ICacheEntry {
|
|||
public function __clone() {
|
||||
$this->data = array_merge([], $this->data);
|
||||
}
|
||||
|
||||
public function getUnencryptedSize(): int {
|
||||
if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) {
|
||||
return $this->data['unencrypted_size'];
|
||||
} else {
|
||||
return $this->data['size'];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -24,6 +24,7 @@
|
|||
|
||||
namespace OC\Files\Cache;
|
||||
|
||||
use OC\Files\Storage\Wrapper\Encryption;
|
||||
use OCP\DB\QueryBuilder\IQueryBuilder;
|
||||
use OCP\Files\Cache\IPropagator;
|
||||
use OCP\Files\Storage\IReliableEtagStorage;
|
||||
|
@ -113,6 +114,20 @@ class Propagator implements IPropagator {
|
|||
->andWhere($builder->expr()->in('path_hash', $hashParams))
|
||||
->andWhere($builder->expr()->gt('size', $builder->expr()->literal(-1, IQueryBuilder::PARAM_INT)));
|
||||
|
||||
if ($this->storage->instanceOfStorage(Encryption::class)) {
|
||||
// in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size
|
||||
$eq = $builder->expr()->eq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT));
|
||||
$sizeColumn = $builder->getColumnName('size');
|
||||
$unencryptedSizeColumn = $builder->getColumnName('unencrypted_size');
|
||||
$builder->set('unencrypted_size', $builder->func()->greatest(
|
||||
$builder->func()->add(
|
||||
$builder->createFunction("CASE WHEN $eq THEN $unencryptedSizeColumn ELSE $sizeColumn END"),
|
||||
$builder->createNamedParameter($sizeDifference)
|
||||
),
|
||||
$builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT)
|
||||
));
|
||||
}
|
||||
|
||||
$builder->execute();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -212,7 +212,12 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
|
|||
public function getSize($includeMounts = true) {
|
||||
if ($includeMounts) {
|
||||
$this->updateEntryfromSubMounts();
|
||||
return isset($this->data['size']) ? 0 + $this->data['size'] : 0;
|
||||
|
||||
if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) {
|
||||
return $this->data['unencrypted_size'];
|
||||
} else {
|
||||
return isset($this->data['size']) ? 0 + $this->data['size'] : 0;
|
||||
}
|
||||
} else {
|
||||
return $this->rawSize;
|
||||
}
|
||||
|
@ -390,7 +395,19 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
|
|||
* @param string $entryPath full path of the child entry
|
||||
*/
|
||||
public function addSubEntry($data, $entryPath) {
|
||||
$this->data['size'] += isset($data['size']) ? $data['size'] : 0;
|
||||
if (!$data) {
|
||||
return;
|
||||
}
|
||||
$hasUnencryptedSize = isset($data['unencrypted_size']) && $data['unencrypted_size'] > 0;
|
||||
if ($hasUnencryptedSize) {
|
||||
$subSize = $data['unencrypted_size'];
|
||||
} else {
|
||||
$subSize = $data['size'] ?: 0;
|
||||
}
|
||||
$this->data['size'] += $subSize;
|
||||
if ($hasUnencryptedSize) {
|
||||
$this->data['unencrypted_size'] += $subSize;
|
||||
}
|
||||
if (isset($data['mtime'])) {
|
||||
$this->data['mtime'] = max($this->data['mtime'], $data['mtime']);
|
||||
}
|
||||
|
|
|
@ -33,6 +33,7 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>
|
||||
*
|
||||
*/
|
||||
|
||||
namespace OC\Files\Storage\Wrapper;
|
||||
|
||||
use OC\Encryption\Exceptions\ModuleDoesNotExistsException;
|
||||
|
@ -41,6 +42,7 @@ use OC\Encryption\Util;
|
|||
use OC\Files\Cache\CacheEntry;
|
||||
use OC\Files\Filesystem;
|
||||
use OC\Files\Mount\Manager;
|
||||
use OC\Files\ObjectStore\ObjectStoreStorage;
|
||||
use OC\Files\Storage\LocalTempFileTrait;
|
||||
use OC\Memcache\ArrayCache;
|
||||
use OCP\Encryption\Exceptions\GenericEncryptionException;
|
||||
|
@ -139,28 +141,36 @@ class Encryption extends Wrapper {
|
|||
$size = $this->unencryptedSize[$fullPath];
|
||||
// update file cache
|
||||
if ($info instanceof ICacheEntry) {
|
||||
$info = $info->getData();
|
||||
$info['encrypted'] = $info['encryptedVersion'];
|
||||
} else {
|
||||
if (!is_array($info)) {
|
||||
$info = [];
|
||||
}
|
||||
$info['encrypted'] = true;
|
||||
$info = new CacheEntry($info);
|
||||
}
|
||||
|
||||
$info['size'] = $size;
|
||||
$this->getCache()->put($path, $info);
|
||||
if ($size !== $info->getUnencryptedSize()) {
|
||||
$this->getCache()->update($info->getId(), [
|
||||
'unencrypted_size' => $size
|
||||
]);
|
||||
}
|
||||
|
||||
return $size;
|
||||
}
|
||||
|
||||
if (isset($info['fileid']) && $info['encrypted']) {
|
||||
return $this->verifyUnencryptedSize($path, $info['size']);
|
||||
return $this->verifyUnencryptedSize($path, $info->getUnencryptedSize());
|
||||
}
|
||||
|
||||
return $this->storage->filesize($path);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $path
|
||||
* @param array $data
|
||||
* @return array
|
||||
*/
|
||||
private function modifyMetaData(string $path, array $data): array {
|
||||
$fullPath = $this->getFullPath($path);
|
||||
$info = $this->getCache()->get($path);
|
||||
|
@ -170,7 +180,7 @@ class Encryption extends Wrapper {
|
|||
$data['size'] = $this->unencryptedSize[$fullPath];
|
||||
} else {
|
||||
if (isset($info['fileid']) && $info['encrypted']) {
|
||||
$data['size'] = $this->verifyUnencryptedSize($path, $info['size']);
|
||||
$data['size'] = $this->verifyUnencryptedSize($path, $info->getUnencryptedSize());
|
||||
$data['encrypted'] = true;
|
||||
}
|
||||
}
|
||||
|
@ -478,7 +488,7 @@ class Encryption extends Wrapper {
|
|||
*
|
||||
* @return int unencrypted size
|
||||
*/
|
||||
protected function verifyUnencryptedSize($path, $unencryptedSize) {
|
||||
protected function verifyUnencryptedSize(string $path, int $unencryptedSize): int {
|
||||
$size = $this->storage->filesize($path);
|
||||
$result = $unencryptedSize;
|
||||
|
||||
|
@ -510,7 +520,7 @@ class Encryption extends Wrapper {
|
|||
*
|
||||
* @return int calculated unencrypted size
|
||||
*/
|
||||
protected function fixUnencryptedSize($path, $size, $unencryptedSize) {
|
||||
protected function fixUnencryptedSize(string $path, int $size, int $unencryptedSize): int {
|
||||
$headerSize = $this->getHeaderSize($path);
|
||||
$header = $this->getHeader($path);
|
||||
$encryptionModule = $this->getEncryptionModule($path);
|
||||
|
@ -581,7 +591,9 @@ class Encryption extends Wrapper {
|
|||
$cache = $this->storage->getCache();
|
||||
if ($cache) {
|
||||
$entry = $cache->get($path);
|
||||
$cache->update($entry['fileid'], ['size' => $newUnencryptedSize]);
|
||||
$cache->update($entry['fileid'], [
|
||||
'unencrypted_size' => $newUnencryptedSize
|
||||
]);
|
||||
}
|
||||
|
||||
return $newUnencryptedSize;
|
||||
|
@ -621,7 +633,12 @@ class Encryption extends Wrapper {
|
|||
* @param bool $preserveMtime
|
||||
* @return bool
|
||||
*/
|
||||
public function moveFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = true) {
|
||||
public function moveFromStorage(
|
||||
Storage\IStorage $sourceStorage,
|
||||
$sourceInternalPath,
|
||||
$targetInternalPath,
|
||||
$preserveMtime = true
|
||||
) {
|
||||
if ($sourceStorage === $this) {
|
||||
return $this->rename($sourceInternalPath, $targetInternalPath);
|
||||
}
|
||||
|
@ -656,7 +673,13 @@ class Encryption extends Wrapper {
|
|||
* @param bool $isRename
|
||||
* @return bool
|
||||
*/
|
||||
public function copyFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = false, $isRename = false) {
|
||||
public function copyFromStorage(
|
||||
Storage\IStorage $sourceStorage,
|
||||
$sourceInternalPath,
|
||||
$targetInternalPath,
|
||||
$preserveMtime = false,
|
||||
$isRename = false
|
||||
) {
|
||||
|
||||
// TODO clean this up once the underlying moveFromStorage in OC\Files\Storage\Wrapper\Common is fixed:
|
||||
// - call $this->storage->copyFromStorage() instead of $this->copyBetweenStorage
|
||||
|
@ -676,7 +699,13 @@ class Encryption extends Wrapper {
|
|||
* @param bool $isRename
|
||||
* @param bool $keepEncryptionVersion
|
||||
*/
|
||||
private function updateEncryptedVersion(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, $keepEncryptionVersion) {
|
||||
private function updateEncryptedVersion(
|
||||
Storage\IStorage $sourceStorage,
|
||||
$sourceInternalPath,
|
||||
$targetInternalPath,
|
||||
$isRename,
|
||||
$keepEncryptionVersion
|
||||
) {
|
||||
$isEncrypted = $this->encryptionManager->isEnabled() && $this->shouldEncrypt($targetInternalPath);
|
||||
$cacheInformation = [
|
||||
'encrypted' => $isEncrypted,
|
||||
|
@ -725,7 +754,13 @@ class Encryption extends Wrapper {
|
|||
* @return bool
|
||||
* @throws \Exception
|
||||
*/
|
||||
private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime, $isRename) {
|
||||
private function copyBetweenStorage(
|
||||
Storage\IStorage $sourceStorage,
|
||||
$sourceInternalPath,
|
||||
$targetInternalPath,
|
||||
$preserveMtime,
|
||||
$isRename
|
||||
) {
|
||||
|
||||
// for versions we have nothing to do, because versions should always use the
|
||||
// key from the original file. Just create a 1:1 copy and done
|
||||
|
@ -743,7 +778,7 @@ class Encryption extends Wrapper {
|
|||
if (isset($info['encrypted']) && $info['encrypted'] === true) {
|
||||
$this->updateUnencryptedSize(
|
||||
$this->getFullPath($targetInternalPath),
|
||||
$info['size']
|
||||
$info->getUnencryptedSize()
|
||||
);
|
||||
}
|
||||
$this->updateEncryptedVersion($sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, true);
|
||||
|
@ -808,13 +843,6 @@ class Encryption extends Wrapper {
|
|||
return (bool)$result;
|
||||
}
|
||||
|
||||
/**
|
||||
* get the path to a local version of the file.
|
||||
* The local version of the file can be temporary and doesn't have to be persistent across requests
|
||||
*
|
||||
* @param string $path
|
||||
* @return string
|
||||
*/
|
||||
public function getLocalFile($path) {
|
||||
if ($this->encryptionManager->isEnabled()) {
|
||||
$cachedFile = $this->getCachedFile($path);
|
||||
|
@ -825,11 +853,6 @@ class Encryption extends Wrapper {
|
|||
return $this->storage->getLocalFile($path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the wrapped storage's value for isLocal()
|
||||
*
|
||||
* @return bool wrapped storage's isLocal() value
|
||||
*/
|
||||
public function isLocal() {
|
||||
if ($this->encryptionManager->isEnabled()) {
|
||||
return false;
|
||||
|
@ -837,15 +860,11 @@ class Encryption extends Wrapper {
|
|||
return $this->storage->isLocal();
|
||||
}
|
||||
|
||||
/**
|
||||
* see https://www.php.net/manual/en/function.stat.php
|
||||
* only the following keys are required in the result: size and mtime
|
||||
*
|
||||
* @param string $path
|
||||
* @return array
|
||||
*/
|
||||
public function stat($path) {
|
||||
$stat = $this->storage->stat($path);
|
||||
if (!$stat) {
|
||||
return false;
|
||||
}
|
||||
$fileSize = $this->filesize($path);
|
||||
$stat['size'] = $fileSize;
|
||||
$stat[7] = $fileSize;
|
||||
|
@ -853,14 +872,6 @@ class Encryption extends Wrapper {
|
|||
return $stat;
|
||||
}
|
||||
|
||||
/**
|
||||
* see https://www.php.net/manual/en/function.hash.php
|
||||
*
|
||||
* @param string $type
|
||||
* @param string $path
|
||||
* @param bool $raw
|
||||
* @return string
|
||||
*/
|
||||
public function hash($type, $path, $raw = false) {
|
||||
$fh = $this->fopen($path, 'rb');
|
||||
$ctx = hash_init($type);
|
||||
|
@ -1068,6 +1079,13 @@ class Encryption extends Wrapper {
|
|||
[$count, $result] = \OC_Helper::streamCopy($stream, $target);
|
||||
fclose($stream);
|
||||
fclose($target);
|
||||
|
||||
// object store, stores the size after write and doesn't update this during scan
|
||||
// manually store the unencrypted size
|
||||
if ($result && $this->getWrapperStorage()->instanceOfStorage(ObjectStoreStorage::class)) {
|
||||
$this->getCache()->put($path, ['unencrypted_size' => $count]);
|
||||
}
|
||||
|
||||
return $count;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -162,4 +162,14 @@ interface ICacheEntry extends ArrayAccess {
|
|||
* @since 18.0.0
|
||||
*/
|
||||
public function getUploadTime(): ?int;
|
||||
|
||||
/**
|
||||
* Get the unencrypted size
|
||||
*
|
||||
* This might be different from the result of getSize
|
||||
*
|
||||
* @return int
|
||||
* @since 25.0.0
|
||||
*/
|
||||
public function getUnencryptedSize(): int;
|
||||
}
|
||||
|
|
|
@ -176,7 +176,7 @@ class S3Test extends ObjectStoreTest {
|
|||
|
||||
// end of file reached
|
||||
fseek($result, $size);
|
||||
self:self::assertTrue(feof($result));
|
||||
self::assertTrue(feof($result));
|
||||
|
||||
$this->assertNoUpload('testfilesizes');
|
||||
}
|
||||
|
|
|
@ -5,6 +5,7 @@ namespace Test\Files\Storage\Wrapper;
|
|||
use OC\Encryption\Exceptions\ModuleDoesNotExistsException;
|
||||
use OC\Encryption\Update;
|
||||
use OC\Encryption\Util;
|
||||
use OC\Files\Cache\CacheEntry;
|
||||
use OC\Files\Storage\Temporary;
|
||||
use OC\Files\Storage\Wrapper\Encryption;
|
||||
use OC\Files\View;
|
||||
|
@ -259,7 +260,7 @@ class EncryptionTest extends Storage {
|
|||
->method('get')
|
||||
->willReturnCallback(
|
||||
function ($path) use ($encrypted) {
|
||||
return ['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1];
|
||||
return new CacheEntry(['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1]);
|
||||
}
|
||||
);
|
||||
|
||||
|
@ -332,7 +333,7 @@ class EncryptionTest extends Storage {
|
|||
->disableOriginalConstructor()->getMock();
|
||||
$cache->expects($this->any())
|
||||
->method('get')
|
||||
->willReturn(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1]);
|
||||
->willReturn(new CacheEntry(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1]));
|
||||
|
||||
$this->instance = $this->getMockBuilder('\OC\Files\Storage\Wrapper\Encryption')
|
||||
->setConstructorArgs(
|
||||
|
@ -910,7 +911,7 @@ class EncryptionTest extends Storage {
|
|||
if ($copyResult) {
|
||||
$cache->expects($this->once())->method('get')
|
||||
->with($sourceInternalPath)
|
||||
->willReturn(['encrypted' => $encrypted, 'size' => 42]);
|
||||
->willReturn(new CacheEntry(['encrypted' => $encrypted, 'size' => 42]));
|
||||
if ($encrypted) {
|
||||
$instance->expects($this->once())->method('updateUnencryptedSize')
|
||||
->with($mountPoint . $targetInternalPath, 42);
|
||||
|
|
|
@ -104,6 +104,9 @@ class HelperStorageTest extends \Test\TestCase {
|
|||
$extStorage->file_put_contents('extfile.txt', 'abcdefghijklmnopq');
|
||||
$extStorage->getScanner()->scan(''); // update root size
|
||||
|
||||
$config = \OC::$server->getConfig();
|
||||
$config->setSystemValue('quota_include_external_storage', false);
|
||||
|
||||
\OC\Files\Filesystem::mount($extStorage, [], '/' . $this->user . '/files/ext');
|
||||
|
||||
$storageInfo = \OC_Helper::getStorageInfo('');
|
||||
|
|
Loading…
Reference in New Issue