mirror of https://github.com/nextcloud/server
Limit-number-of-concurrent-preview-generations
Signed-off-by: Bowen Ding <dbw9580@live.com> Signed-off-by: szaimen <szaimen@e.mail.de>
This commit is contained in:
parent
c88aabd125
commit
f9e9cd270d
|
@ -717,6 +717,11 @@ Raw output
|
|||
$recommendedPHPModules[] = 'intl';
|
||||
}
|
||||
|
||||
if (!extension_loaded('sysvsem')) {
|
||||
// used to limit the usage of resources by preview generator
|
||||
$recommendedPHPModules[] = 'sysvsem';
|
||||
}
|
||||
|
||||
if (!defined('PASSWORD_ARGON2I') && PHP_VERSION_ID >= 70400) {
|
||||
// Installing php-sodium on >=php7.4 will provide PASSWORD_ARGON2I
|
||||
// on previous version argon2 wasn't part of the "standard" extension
|
||||
|
|
|
@ -1118,6 +1118,28 @@ $CONFIG = [
|
|||
* Defaults to ``true``
|
||||
*/
|
||||
'enable_previews' => true,
|
||||
|
||||
/**
|
||||
* Number of all preview requests being processed concurrently,
|
||||
* including previews that need to be newly generated, and those that have
|
||||
* been generated.
|
||||
*
|
||||
* This should be greater than 'preview_concurrency_new'.
|
||||
* If unspecified, defaults to twice the value of 'preview_concurrency_new'.
|
||||
*/
|
||||
'preview_concurrency_all' => 8,
|
||||
|
||||
/**
|
||||
* Number of new previews that are being concurrently generated.
|
||||
*
|
||||
* Depending on the max preview size set by 'preview_max_x' and 'preview_max_y',
|
||||
* the generation process can consume considerable CPU and memory resources.
|
||||
* It's recommended to limit this to be no greater than the number of CPU cores.
|
||||
* If unspecified, defaults to the number of CPU cores, or 4 if that cannot
|
||||
* be determined.
|
||||
*/
|
||||
'preview_concurrency_new' => 4,
|
||||
|
||||
/**
|
||||
* The maximum width, in pixels, of a preview. A value of ``null`` means there
|
||||
* is no limit.
|
||||
|
|
|
@ -48,6 +48,8 @@ use Symfony\Component\EventDispatcher\EventDispatcherInterface;
|
|||
use Symfony\Component\EventDispatcher\GenericEvent;
|
||||
|
||||
class Generator {
|
||||
public const SEMAPHORE_ID_ALL = 0x0a11;
|
||||
public const SEMAPHORE_ID_NEW = 0x07ea;
|
||||
|
||||
/** @var IPreview */
|
||||
private $previewManager;
|
||||
|
@ -302,6 +304,98 @@ class Generator {
|
|||
throw new NotFoundException('No provider successfully handled the preview generation');
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire a semaphore of the specified id and concurrency, blocking if necessary.
|
||||
* Return an identifier of the semaphore on success, which can be used to release it via
|
||||
* {@see Generator::unguardWithSemaphore()}.
|
||||
*
|
||||
* @param int $semId
|
||||
* @param int $concurrency
|
||||
* @return false|resource the semaphore on success or false on failure
|
||||
*/
|
||||
public static function guardWithSemaphore(int $semId, int $concurrency) {
|
||||
if (!extension_loaded('sysvsem')) {
|
||||
return false;
|
||||
}
|
||||
$sem = sem_get($semId, $concurrency);
|
||||
if ($sem === false) {
|
||||
return false;
|
||||
}
|
||||
if (!sem_acquire($sem)) {
|
||||
return false;
|
||||
}
|
||||
return $sem;
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases the semaphore acquired from {@see Generator::guardWithSemaphore()}.
|
||||
*
|
||||
* @param resource|bool $semId the semaphore identifier returned by guardWithSemaphore
|
||||
* @return bool
|
||||
*/
|
||||
public static function unguardWithSemaphore($semId): bool {
|
||||
if (!is_resource($semId) || !extension_loaded('sysvsem')) {
|
||||
return false;
|
||||
}
|
||||
return sem_release($semId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of concurrent threads supported by the host.
|
||||
*
|
||||
* @return int number of concurrent threads, or 0 if it cannot be determined
|
||||
*/
|
||||
public static function getHardwareConcurrency(): int {
|
||||
static $width;
|
||||
if (!isset($width)) {
|
||||
if (is_file("/proc/cpuinfo")) {
|
||||
$width = substr_count(file_get_contents("/proc/cpuinfo"), "processor");
|
||||
} else {
|
||||
$width = 0;
|
||||
}
|
||||
}
|
||||
return $width;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get number of concurrent preview generations from system config
|
||||
*
|
||||
* Two config entries, `preview_concurrency_new` and `preview_concurrency_all`,
|
||||
* are available. If not set, the default values are determined with the hardware concurrency
|
||||
* of the host. In case the hardware concurrency cannot be determined, or the user sets an
|
||||
* invalid value, fallback values are:
|
||||
* For new images whose previews do not exist and need to be generated, 4;
|
||||
* For all preview generation requests, 8.
|
||||
* Value of `preview_concurrency_all` should be greater than or equal to that of
|
||||
* `preview_concurrency_new`, otherwise, the latter is returned.
|
||||
*
|
||||
* @param string $type either `preview_concurrency_new` or `preview_concurrency_all`
|
||||
* @return int number of concurrent preview generations, or -1 if $type is invalid
|
||||
*/
|
||||
public function getNumConcurrentPreviews(string $type): int {
|
||||
static $cached = array();
|
||||
if (array_key_exists($type, $cached)) {
|
||||
return $cached[$type];
|
||||
}
|
||||
|
||||
$hardwareConcurrency = self::getHardwareConcurrency();
|
||||
switch ($type) {
|
||||
case "preview_concurrency_all":
|
||||
$fallback = $hardwareConcurrency > 0 ? $hardwareConcurrency * 2 : 8;
|
||||
$concurrency_all = $this->config->getSystemValueInt($type, $fallback);
|
||||
$concurrency_new = $this->getNumConcurrentPreviews("preview_concurrency_new");
|
||||
$cached[$type] = max($concurrency_all, $concurrency_new);
|
||||
break;
|
||||
case "preview_concurrency_new":
|
||||
$fallback = $hardwareConcurrency > 0 ? $hardwareConcurrency : 4;
|
||||
$cached[$type] = $this->config->getSystemValueInt($type, $fallback);
|
||||
break;
|
||||
default:
|
||||
return -1;
|
||||
}
|
||||
return $cached[$type];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ISimpleFolder $previewFolder
|
||||
* @param File $file
|
||||
|
@ -340,7 +434,13 @@ class Generator {
|
|||
$maxWidth = $this->config->getSystemValueInt('preview_max_x', 4096);
|
||||
$maxHeight = $this->config->getSystemValueInt('preview_max_y', 4096);
|
||||
|
||||
$preview = $this->helper->getThumbnail($provider, $file, $maxWidth, $maxHeight);
|
||||
$previewConcurrency = $this->getNumConcurrentPreviews('preview_concurrency_new');
|
||||
$sem = self::guardWithSemaphore(self::SEMAPHORE_ID_NEW, $previewConcurrency);
|
||||
try {
|
||||
$preview = $this->helper->getThumbnail($provider, $file, $maxWidth, $maxHeight);
|
||||
} finally {
|
||||
self::unguardWithSemaphore($sem);
|
||||
}
|
||||
|
||||
if (!($preview instanceof IImage)) {
|
||||
continue;
|
||||
|
@ -510,29 +610,34 @@ class Generator {
|
|||
throw new \InvalidArgumentException('Failed to generate preview, failed to load image');
|
||||
}
|
||||
|
||||
if ($crop) {
|
||||
if ($height !== $preview->height() && $width !== $preview->width()) {
|
||||
//Resize
|
||||
$widthR = $preview->width() / $width;
|
||||
$heightR = $preview->height() / $height;
|
||||
$previewConcurrency = $this->getNumConcurrentPreviews('preview_concurrency_new');
|
||||
$sem = self::guardWithSemaphore(self::SEMAPHORE_ID_NEW, $previewConcurrency);
|
||||
try {
|
||||
if ($crop) {
|
||||
if ($height !== $preview->height() && $width !== $preview->width()) {
|
||||
//Resize
|
||||
$widthR = $preview->width() / $width;
|
||||
$heightR = $preview->height() / $height;
|
||||
|
||||
if ($widthR > $heightR) {
|
||||
$scaleH = $height;
|
||||
$scaleW = $maxWidth / $heightR;
|
||||
} else {
|
||||
$scaleH = $maxHeight / $widthR;
|
||||
$scaleW = $width;
|
||||
if ($widthR > $heightR) {
|
||||
$scaleH = $height;
|
||||
$scaleW = $maxWidth / $heightR;
|
||||
} else {
|
||||
$scaleH = $maxHeight / $widthR;
|
||||
$scaleW = $width;
|
||||
}
|
||||
$preview = $preview->preciseResizeCopy((int)round($scaleW), (int)round($scaleH));
|
||||
}
|
||||
$preview = $preview->preciseResizeCopy((int)round($scaleW), (int)round($scaleH));
|
||||
$cropX = (int)floor(abs($width - $preview->width()) * 0.5);
|
||||
$cropY = (int)floor(abs($height - $preview->height()) * 0.5);
|
||||
$preview = $preview->cropCopy($cropX, $cropY, $width, $height);
|
||||
} else {
|
||||
$preview = $maxPreview->resizeCopy(max($width, $height));
|
||||
}
|
||||
$cropX = (int)floor(abs($width - $preview->width()) * 0.5);
|
||||
$cropY = (int)floor(abs($height - $preview->height()) * 0.5);
|
||||
$preview = $preview->cropCopy($cropX, $cropY, $width, $height);
|
||||
} else {
|
||||
$preview = $maxPreview->resizeCopy(max($width, $height));
|
||||
} finally {
|
||||
self::unguardWithSemaphore($sem);
|
||||
}
|
||||
|
||||
|
||||
$path = $this->generatePath($width, $height, $crop, $preview->dataMimeType(), $prefix);
|
||||
try {
|
||||
$file = $previewFolder->newFile($path);
|
||||
|
|
|
@ -182,7 +182,15 @@ class PreviewManager implements IPreview {
|
|||
* @since 11.0.0 - \InvalidArgumentException was added in 12.0.0
|
||||
*/
|
||||
public function getPreview(File $file, $width = -1, $height = -1, $crop = false, $mode = IPreview::MODE_FILL, $mimeType = null) {
|
||||
return $this->getGenerator()->getPreview($file, $width, $height, $crop, $mode, $mimeType);
|
||||
$previewConcurrency = $this->getGenerator()->getNumConcurrentPreviews('preview_concurrency_all');
|
||||
$sem = Generator::guardWithSemaphore(Generator::SEMAPHORE_ID_ALL, $previewConcurrency);
|
||||
try {
|
||||
$preview = $this->getGenerator()->getPreview($file, $width, $height, $crop, $mode, $mimeType);
|
||||
} finally {
|
||||
Generator::unguardWithSemaphore($sem);
|
||||
}
|
||||
|
||||
return $preview;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -158,8 +158,13 @@ class GeneratorTest extends \Test\TestCase {
|
|||
->willReturn($previewFolder);
|
||||
|
||||
$this->config->method('getSystemValue')
|
||||
->willReturnCallback(function ($key, $defult) {
|
||||
return $defult;
|
||||
->willReturnCallback(function ($key, $default) {
|
||||
return $default;
|
||||
});
|
||||
|
||||
$this->config->method('getSystemValueInt')
|
||||
->willReturnCallback(function ($key, $default) {
|
||||
return $default;
|
||||
});
|
||||
|
||||
$invalidProvider = $this->createMock(IProviderV2::class);
|
||||
|
|
Loading…
Reference in New Issue