nextcloud
/
server
mirror of https://github.com/nextcloud/server
0
0
Fork 0
Code Issues Releases Wiki Activity

optimize query pattern used by storage filter 

Signed-off-by: Robin Appelman <robin@icewind.nl>
This commit is contained in:
Robin Appelman 2023-09-21 13:49:16 +02:00
parent 1f0cba5f99
commit 2e14a7a4a6
No known key found for this signature in database
GPG Key ID: 42B69D8A64526EFB
18 changed files with 731 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/composer/composer/LICENSE
View File

@ -1,4 +1,3 @@
Copyright (c) Nils Adermann, Jordi Boggiano
Permission is hereby granted, free of charge, to any person obtaining a copy
@ -18,4 +17,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

5
lib/composer/composer/autoload_classmap.php
View File

@ -1406,9 +1406,14 @@ return array(
'OC\\Files\\ObjectStore\\Swift' => $baseDir . '/lib/private/Files/ObjectStore/Swift.php',
'OC\\Files\\ObjectStore\\SwiftFactory' => $baseDir . '/lib/private/Files/ObjectStore/SwiftFactory.php',
'OC\\Files\\ObjectStore\\SwiftV2CachingAuthService' => $baseDir . '/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php',
'OC\\Files\\Search\\QueryOptimizer\\FlattenNestedBool' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php',
'OC\\Files\\Search\\QueryOptimizer\\FlattenSingleArgumentBinaryOperation' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php',
'OC\\Files\\Search\\QueryOptimizer\\MergeDistributiveOperations' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php',
'OC\\Files\\Search\\QueryOptimizer\\OrEqualsToIn' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php',
'OC\\Files\\Search\\QueryOptimizer\\PathPrefixOptimizer' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php',
'OC\\Files\\Search\\QueryOptimizer\\QueryOptimizer' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php',
'OC\\Files\\Search\\QueryOptimizer\\QueryOptimizerStep' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/QueryOptimizerStep.php',
'OC\\Files\\Search\\QueryOptimizer\\ReplacingOptimizerStep' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php',
'OC\\Files\\Search\\SearchBinaryOperator' => $baseDir . '/lib/private/Files/Search/SearchBinaryOperator.php',
'OC\\Files\\Search\\SearchComparison' => $baseDir . '/lib/private/Files/Search/SearchComparison.php',
'OC\\Files\\Search\\SearchOrder' => $baseDir . '/lib/private/Files/Search/SearchOrder.php',

5
lib/composer/composer/autoload_static.php
View File

@ -1439,9 +1439,14 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OC\\Files\\ObjectStore\\Swift' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Swift.php',
'OC\\Files\\ObjectStore\\SwiftFactory' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftFactory.php',
'OC\\Files\\ObjectStore\\SwiftV2CachingAuthService' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php',
'OC\\Files\\Search\\QueryOptimizer\\FlattenNestedBool' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php',
'OC\\Files\\Search\\QueryOptimizer\\FlattenSingleArgumentBinaryOperation' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php',
'OC\\Files\\Search\\QueryOptimizer\\MergeDistributiveOperations' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php',
'OC\\Files\\Search\\QueryOptimizer\\OrEqualsToIn' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php',
'OC\\Files\\Search\\QueryOptimizer\\PathPrefixOptimizer' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php',
'OC\\Files\\Search\\QueryOptimizer\\QueryOptimizer' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php',
'OC\\Files\\Search\\QueryOptimizer\\QueryOptimizerStep' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/QueryOptimizerStep.php',
'OC\\Files\\Search\\QueryOptimizer\\ReplacingOptimizerStep' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php',
'OC\\Files\\Search\\SearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchBinaryOperator.php',
'OC\\Files\\Search\\SearchComparison' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchComparison.php',
'OC\\Files\\Search\\SearchOrder' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchOrder.php',

128
lib/private/Files/Cache/SearchBuilder.php
View File

@ -48,6 +48,7 @@ class SearchBuilder {
ISearchComparison::COMPARE_LESS_THAN => 'lt',
ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lte',
ISearchComparison::COMPARE_DEFINED => 'isNotNull',
ISearchComparison::COMPARE_IN => 'in',
];
protected static $searchOperatorNegativeMap = [
@ -59,6 +60,34 @@ class SearchBuilder {
ISearchComparison::COMPARE_LESS_THAN => 'gte',
ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'gt',
ISearchComparison::COMPARE_DEFINED => 'isNull',
ISearchComparison::COMPARE_IN => 'notIn',
];
protected static $fieldTypes = [
'mimetype' => 'string',
'mtime' => 'integer',
'name' => 'string',
'path' => 'string',
'size' => 'integer',
'tagname' => 'string',
'systemtag' => 'string',
'favorite' => 'boolean',
'fileid' => 'integer',
'storage' => 'integer',
'share_with' => 'string',
'share_type' => 'integer',
'owner' => 'string',
];
protected static $paramTypeMap = [
'string' => IQueryBuilder::PARAM_STR,
'integer' => IQueryBuilder::PARAM_INT,
'boolean' => IQueryBuilder::PARAM_BOOL,
];
protected static $paramArrayTypeMap = [
'string' => IQueryBuilder::PARAM_STR_ARRAY,
'integer' => IQueryBuilder::PARAM_INT_ARRAY,
'boolean' => IQueryBuilder::PARAM_INT_ARRAY,
];
public const TAG_FAVORITE = '_$!<Favorite>!$_';
@ -142,31 +171,56 @@ class SearchBuilder {
?IMetadataQuery $metadataQuery = null
) {
if ($comparison->getExtra()) {
[$field, $value, $type] = $this->getExtraOperatorField($comparison, $metadataQuery);
[$field, $value, $type, $paramType] = $this->getExtraOperatorField($comparison, $metadataQuery);
} else {
[$field, $value, $type] = $this->getOperatorFieldAndValue($comparison);
[$field, $value, $type, $paramType] = $this->getOperatorFieldAndValue($comparison);
}
if (isset($operatorMap[$type])) {
$queryOperator = $operatorMap[$type];
return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value));
return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value, $paramType));
} else {
throw new \InvalidArgumentException('Invalid operator type: ' . $comparison->getType());
}
}
private function getOperatorFieldAndValue(ISearchComparison $operator) {
/**
* @param ISearchComparison $operator
* @return list{string, string|integer|\DateTime|(\DateTime|int|string)[], string, string}
*/
private function getOperatorFieldAndValue(ISearchComparison $operator): array {
$this->validateComparison($operator);
$field = $operator->getField();
$value = $operator->getValue();
$type = $operator->getType();
$pathEqHash = $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true);
return $this->getOperatorFieldAndValueInner($field, $value, $type, $pathEqHash);
}
/**
* @param string $field
* @param string|integer|\DateTime|(\DateTime|int|string)[] $value
* @param string $type
* @return list{string, string|integer|\DateTime|(\DateTime|int|string)[], string, string}
*/
private function getOperatorFieldAndValueInner(string $field, mixed $value, string $type, bool $pathEqHash): array {
$paramType = self::$fieldTypes[$field];
if ($type === ISearchComparison::COMPARE_IN) {
$resultField = $field;
$values = [];
foreach ($value as $arrayValue) {
/** @var string|integer|\DateTime $arrayValue */
[$arrayField, $arrayValue] = $this->getOperatorFieldAndValueInner($field, $arrayValue, ISearchComparison::COMPARE_EQUAL, $pathEqHash);
$resultField = $arrayField;
$values[] = $arrayValue;
}
return [$resultField, $values, ISearchComparison::COMPARE_IN, $paramType];
}
if ($field === 'mimetype') {
$value = (string)$value;
if ($operator->getType() === ISearchComparison::COMPARE_EQUAL) {
if ($type === ISearchComparison::COMPARE_EQUAL) {
$value = (int)$this->mimetypeLoader->getId($value);
} elseif ($operator->getType() === ISearchComparison::COMPARE_LIKE) {
} elseif ($type === ISearchComparison::COMPARE_LIKE) {
// transform "mimetype='foo/%'" to "mimepart='foo'"
if (preg_match('|(.+)/%|', $value, $matches)) {
$field = 'mimepart';
@ -183,6 +237,7 @@ class SearchBuilder {
} elseif ($field === 'favorite') {
$field = 'tag.category';
$value = self::TAG_FAVORITE;
$paramType = 'string';
} elseif ($field === 'name') {
$field = 'file.name';
} elseif ($field === 'tagname') {
@ -191,53 +246,49 @@ class SearchBuilder {
$field = 'systemtag.name';
} elseif ($field === 'fileid') {
$field = 'file.fileid';
} elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true)) {
} elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $pathEqHash) {
$field = 'path_hash';
$value = md5((string)$value);
} elseif ($field === 'owner') {
$field = 'uid_owner';
}
return [$field, $value, $type];
return [$field, $value, $type, $paramType];
}
private function validateComparison(ISearchComparison $operator) {
$types = [
'mimetype' => 'string',
'mtime' => 'integer',
'name' => 'string',
'path' => 'string',
'size' => 'integer',
'tagname' => 'string',
'systemtag' => 'string',
'favorite' => 'boolean',
'fileid' => 'integer',
'storage' => 'integer',
'share_with' => 'string',
'share_type' => 'integer',
'owner' => 'string',
];
$comparisons = [
'mimetype' => ['eq', 'like'],
'mimetype' => ['eq', 'like', 'in'],
'mtime' => ['eq', 'gt', 'lt', 'gte', 'lte'],
'name' => ['eq', 'like', 'clike'],
'path' => ['eq', 'like', 'clike'],
'name' => ['eq', 'like', 'clike', 'in'],
'path' => ['eq', 'like', 'clike', 'in'],
'size' => ['eq', 'gt', 'lt', 'gte', 'lte'],
'tagname' => ['eq', 'like'],
'systemtag' => ['eq', 'like'],
'favorite' => ['eq'],
'fileid' => ['eq'],
'storage' => ['eq'],
'fileid' => ['eq', 'in'],
'storage' => ['eq', 'in'],
'share_with' => ['eq'],
'share_type' => ['eq'],
'owner' => ['eq'],
];
if (!isset($types[$operator->getField()])) {
if (!isset(self::$fieldTypes[$operator->getField()])) {
throw new \InvalidArgumentException('Unsupported comparison field ' . $operator->getField());
}
$type = $types[$operator->getField()];
if (gettype($operator->getValue()) !== $type) {
throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField());
$type = self::$fieldTypes[$operator->getField()];
if ($operator->getType() === ISearchComparison::COMPARE_IN) {
if (!is_array($operator->getValue())) {
throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField());
}
foreach ($operator->getValue() as $arrayValue) {
if (gettype($arrayValue) !== $type) {
throw new \InvalidArgumentException('Invalid type in array for field ' . $operator->getField());
}
}
} else {
if (gettype($operator->getValue()) !== $type) {
throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField());
}
}
if (!in_array($operator->getType(), $comparisons[$operator->getField()])) {
throw new \InvalidArgumentException('Unsupported comparison for field ' . $operator->getField() . ': ' . $operator->getType());
@ -246,6 +297,7 @@ class SearchBuilder {
private function getExtraOperatorField(ISearchComparison $operator, IMetadataQuery $metadataQuery): array {
$paramType = self::$fieldTypes[$field];
$field = $operator->getField();
$value = $operator->getValue();
$type = $operator->getType();
@ -259,17 +311,17 @@ class SearchBuilder {
throw new \InvalidArgumentException('Invalid extra type: ' . $operator->getExtra());
}
return [$field, $value, $type];
return [$field, $value, $type, $paramType];
}
private function getParameterForValue(IQueryBuilder $builder, $value) {
private function getParameterForValue(IQueryBuilder $builder, $value, string $paramType) {
if ($value instanceof \DateTime) {
$value = $value->getTimestamp();
}
if (is_numeric($value)) {
$type = IQueryBuilder::PARAM_INT;
if (is_array($value)) {
$type = self::$paramArrayTypeMap[$paramType];
} else {
$type = IQueryBuilder::PARAM_STR;
$type = self::$paramTypeMap[$paramType];
}
return $builder->createNamedParameter($value, $type);
}

30
lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php Normal file
View File

@ -0,0 +1,30 @@
<?php
namespace OC\Files\Search\QueryOptimizer;
use OC\Files\Search\SearchBinaryOperator;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchOperator;
class FlattenNestedBool extends QueryOptimizerStep {
public function processOperator(ISearchOperator &$operator) {
if (
$operator instanceof SearchBinaryOperator && (
$operator->getType() === ISearchBinaryOperator::OPERATOR_OR ||
$operator->getType() === ISearchBinaryOperator::OPERATOR_AND
)
) {
$newArguments = [];
foreach ($operator->getArguments() as $oldArgument) {
if ($oldArgument instanceof SearchBinaryOperator && $oldArgument->getType() === $operator->getType()) {
$newArguments = array_merge($newArguments, $oldArgument->getArguments());
} else {
$newArguments[] = $oldArgument;
}
}
$operator->setArguments($newArguments);
}
parent::processOperator($operator);
}
}

27
lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php Normal file
View File

@ -0,0 +1,27 @@
<?php
namespace OC\Files\Search\QueryOptimizer;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchOperator;
/**
* replace single argument AND and OR operations with their single argument
*/
class FlattenSingleArgumentBinaryOperation extends ReplacingOptimizerStep {
public function processOperator(ISearchOperator &$operator): bool {
parent::processOperator($operator);
if (
$operator instanceof ISearchBinaryOperator &&
count($operator->getArguments()) === 1 &&
(
$operator->getType() === ISearchBinaryOperator::OPERATOR_OR ||
$operator->getType() === ISearchBinaryOperator::OPERATOR_AND
)
) {
$operator = $operator->getArguments()[0];
return true;
}
return false;
}
}

99
lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php Normal file
View File

@ -0,0 +1,99 @@
<?php
namespace OC\Files\Search\QueryOptimizer;
use OC\Files\Search\SearchBinaryOperator;
use OC\Files\Search\SearchComparison;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchOperator;
/**
* Attempt to transform
*
* (A AND B) OR (A AND C) into A AND (B OR C)
*/
class MergeDistributiveOperations extends ReplacingOptimizerStep {
public function processOperator(ISearchOperator &$operator): bool {
if (
$operator instanceof SearchBinaryOperator &&
$this->isAllSameBinaryOperation($operator->getArguments())
) {
$topLevelType = $operator->getType();
$groups = $this->groupBinaryOperatorsByChild($operator->getArguments(), 0);
$outerOperations = array_map(function (array $operators) use ($topLevelType) {
if (count($operators) === 1) {
return $operators[0];
}
/** @var ISearchBinaryOperator $firstArgument */
$firstArgument = $operators[0];
$outerType = $firstArgument->getType();
$extractedLeftHand = $firstArgument->getArguments()[0];
$rightHandArguments = array_map(function (ISearchOperator $inner) {
/** @var ISearchBinaryOperator $inner */
$arguments = $inner->getArguments();
array_shift($arguments);
if (count($arguments) === 1) {
return $arguments[0];
}
return new SearchBinaryOperator($inner->getType(), $arguments);
}, $operators);
$extractedRightHand = new SearchBinaryOperator($topLevelType, $rightHandArguments);
return new SearchBinaryOperator(
$outerType,
[$extractedLeftHand, $extractedRightHand]
);
}, $groups);
$operator = new SearchBinaryOperator($topLevelType, $outerOperations);
parent::processOperator($operator);
return true;
}
return parent::processOperator($operator);
}
/**
* Check that a list of operators is all the same type of (non-empty) binary operators
*
* @param ISearchOperator[] $operators
* @return bool
* @psalm-assert-if-true SearchBinaryOperator[] $operators
*/
private function isAllSameBinaryOperation(array $operators): bool {
$operation = null;
foreach ($operators as $operator) {
if (!$operator instanceof SearchBinaryOperator) {
return false;
}
if (!$operator->getArguments()) {
return false;
}
if ($operation === null) {
$operation = $operator->getType();
} else {
if ($operation !== $operator->getType()) {
return false;
}
}
}
return true;
}
/**
* Group a list of binary search operators that have a common argument
*
* @param SearchBinaryOperator[] $operators
* @return SearchBinaryOperator[][]
*/
private function groupBinaryOperatorsByChild(array $operators, int $index = 0): array {
$result = [];
foreach ($operators as $operator) {
/** @var SearchBinaryOperator|SearchComparison $child */
$child = $operator->getArguments()[$index];
;
$childKey = (string) $child;
$result[$childKey][] = $operator;
}
return array_values($result);
}
}

68
lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php Normal file
View File

@ -0,0 +1,68 @@
<?php
namespace OC\Files\Search\QueryOptimizer;
use OC\Files\Search\SearchBinaryOperator;
use OC\Files\Search\SearchComparison;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchComparison;
use OCP\Files\Search\ISearchOperator;
/**
* transform (field == A OR field == B ...) into field IN (A, B, ...)
*/
class OrEqualsToIn extends ReplacingOptimizerStep {
public function processOperator(ISearchOperator &$operator): bool {
if (
$operator instanceof ISearchBinaryOperator &&
$operator->getType() === ISearchBinaryOperator::OPERATOR_OR
) {
$groups = $this->groupEqualsComparisonsByField($operator->getArguments());
$newParts = array_map(function (array $group) {
if (count($group) > 1) {
// because of the logic from `groupEqualsComparisonsByField` we now that group is all comparisons on the same field
/** @var ISearchComparison[] $group */
$field = $group[0]->getField();
$values = array_map(function (ISearchComparison $comparison) {
/** @var string|integer|bool|\DateTime $value */
$value = $comparison->getValue();
return $value;
}, $group);
$in = new SearchComparison(ISearchComparison::COMPARE_IN, $field, $values);
$in->setQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, $group[0]->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true));
return $in;
} else {
return $group[0];
}
}, $groups);
if (count($newParts) === 1) {
$operator = $newParts[0];
} else {
$operator = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $newParts);
}
parent::processOperator($operator);
return true;
}
parent::processOperator($operator);
return false;
}
/**
* Non-equals operators are put in a separate group for each
*
* @param ISearchOperator[] $operators
* @return ISearchOperator[][]
*/
private function groupEqualsComparisonsByField(array $operators): array {
$result = [];
foreach ($operators as $operator) {
if ($operator instanceof ISearchComparison && $operator->getType() === ISearchComparison::COMPARE_EQUAL) {
$key = $operator->getField() . $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true);
$result[$key][] = $operator;
} else {
$result[] = [$operator];
}
}
return array_values($result);
}
}

13
lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php
View File

@ -29,15 +29,18 @@ class QueryOptimizer {
/** @var QueryOptimizerStep[] */
private $steps = [];
public function __construct(
PathPrefixOptimizer $pathPrefixOptimizer
) {
public function __construct() {
// note that the order here is relevant
$this->steps = [
$pathPrefixOptimizer
new PathPrefixOptimizer(),
new MergeDistributiveOperations(),
new FlattenSingleArgumentBinaryOperation(),
new OrEqualsToIn(),
new FlattenNestedBool(),
];
}
public function processOperator(ISearchOperator $operator) {
public function processOperator(ISearchOperator &$operator) {
foreach ($this->steps as $step) {
$step->inspectOperator($operator);
}

31
lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php Normal file
View File

@ -0,0 +1,31 @@
<?php
namespace OC\Files\Search\QueryOptimizer;
use OC\Files\Search\SearchBinaryOperator;
use OCP\Files\Search\ISearchOperator;
/**
* Optimizer step that can replace the $operator altogether instead of just modifying it
* These steps need some extra logic to properly replace the arguments of binary operators
*/
class ReplacingOptimizerStep extends QueryOptimizerStep {
/**
* Allow optimizer steps to modify query operators
*
* Returns true if the reference $operator points to a new value
*/
public function processOperator(ISearchOperator &$operator): bool {
if ($operator instanceof SearchBinaryOperator) {
$modified = false;
$arguments = $operator->getArguments();
foreach ($arguments as &$argument) {
$modified = $modified || $this->processOperator($argument);
}
if ($modified) {
$operator->setArguments($arguments);
}
}
return false;
}
}

19
lib/private/Files/Search/SearchBinaryOperator.php
View File

@ -28,7 +28,7 @@ use OCP\Files\Search\ISearchOperator;
class SearchBinaryOperator implements ISearchBinaryOperator {
/** @var string */
private $type;
/** @var ISearchOperator[] */
/** @var (SearchBinaryOperator|SearchComparison)[] */
private $arguments;
private $hints = [];
@ -36,7 +36,7 @@ class SearchBinaryOperator implements ISearchBinaryOperator {
* SearchBinaryOperator constructor.
*
* @param string $type
* @param ISearchOperator[] $arguments
* @param (SearchBinaryOperator|SearchComparison)[] $arguments
*/
public function __construct($type, array $arguments) {
$this->type = $type;
@ -57,6 +57,14 @@ class SearchBinaryOperator implements ISearchBinaryOperator {
return $this->arguments;
}
/**
* @param ISearchOperator[] $arguments
* @return void
*/
public function setArguments(array $arguments): void {
$this->arguments = $arguments;
}
public function getQueryHint(string $name, $default) {
return $this->hints[$name] ?? $default;
}
@ -64,4 +72,11 @@ class SearchBinaryOperator implements ISearchBinaryOperator {
public function setQueryHint(string $name, $value): void {
$this->hints[$name] = $value;
}
public function __toString(): string {
if ($this->type === ISearchBinaryOperator::OPERATOR_NOT) {
return '(not ' . $this->arguments[0] . ')';
}
return '(' . implode(' ' . $this->type . ' ', $this->arguments) . ')';
}
}

10
lib/private/Files/Search/SearchComparison.php
View File

@ -33,7 +33,7 @@ class SearchComparison implements ISearchComparison {
public function __construct(
private string $type,
private string $field,
private \DateTime|int|string|bool $value,
private \DateTime|int|string|bool|array $value,
private string $extra = ''
) {
}
@ -53,9 +53,9 @@ class SearchComparison implements ISearchComparison {
}
/**
* @return \DateTime|int|string|bool
* @return \DateTime|int|string|bool|(\DateTime|int|string)[]
*/
public function getValue(): string|int|bool|\DateTime {
public function getValue(): string|int|bool|\DateTime|array {
return $this->value;
}
@ -78,4 +78,8 @@ class SearchComparison implements ISearchComparison {
public static function escapeLikeParameter(string $param): string {
return addcslashes($param, '\\_%');
}
public function __toString(): string {
return $this->field . ' ' . $this->type . ' ' . json_encode($this->value);
}
}

5
lib/public/Files/Search/ISearchComparison.php
View File

@ -67,6 +67,7 @@ interface ISearchComparison extends ISearchOperator {
* @since 28.0.0
*/
public const COMPARE_DEFINED = 'is-defined';
public const COMPARE_IN = 'in';
/**
* @since 23.0.0
@ -102,8 +103,8 @@ interface ISearchComparison extends ISearchOperator {
/**
* Get the value to compare the field with
*
* @return string|integer|bool|\DateTime
* @return string|integer|bool|\DateTime|(\DateTime|int|string)[]
* @since 12.0.0
*/
public function getValue(): string|int|bool|\DateTime;
public function getValue(): string|int|bool|\DateTime|array;
}

1
tests/lib/Files/Cache/SearchBuilderTest.php
View File

@ -154,6 +154,7 @@ class SearchBuilderTest extends TestCase {
[new SearchComparison(ISearchComparison::COMPARE_LIKE, 'name', 'foo%'), [0, 1]],
[new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', 'image/jpg'), [0]],
[new SearchComparison(ISearchComparison::COMPARE_LIKE, 'mimetype', 'image/%'), [0, 1]],
[new SearchComparison(ISearchComparison::COMPARE_IN, 'mimetype', ['image/jpg', 'image/png']), [0, 1]],
[new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'size', 50),
new SearchComparison(ISearchComparison::COMPARE_LESS_THAN, 'mtime', 125)

45
tests/lib/Files/Search/QueryOptimizer/CombinedTests.php Normal file
View File

@ -0,0 +1,45 @@
<?php
namespace Test\Files\Search\QueryOptimizer;
use OC\Files\Search\QueryOptimizer\QueryOptimizer;
use OC\Files\Search\SearchBinaryOperator;
use OC\Files\Search\SearchComparison;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchComparison;
use Test\TestCase;
class CombinedTests extends TestCase {
private QueryOptimizer $optimizer;
protected function setUp(): void {
parent::setUp();
$this->optimizer = new QueryOptimizer();
}
public function testBasicOrOfAnds() {
$operator = new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_OR,
[
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "foo"),
]),
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "bar"),
]),
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "asd"),
])
]
);
$this->assertEquals('((storage eq 1 and path eq "foo") or (storage eq 1 and path eq "bar") or (storage eq 1 and path eq "asd"))', $operator->__toString());
$this->optimizer->processOperator($operator);
$this->assertEquals('(storage eq 1 and path in ["foo","bar","asd"])', $operator->__toString());
}
}

42
tests/lib/Files/Search/QueryOptimizer/FlattenNestedBoolTest.php Normal file
View File

@ -0,0 +1,42 @@
<?php
namespace Test\Files\Search\QueryOptimizer;
use OC\Files\Search\QueryOptimizer\FlattenNestedBool;
use OC\Files\Search\QueryOptimizer\FlattenSingleArgumentBinaryOperation;
use OC\Files\Search\SearchBinaryOperator;
use OC\Files\Search\SearchComparison;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchComparison;
use Test\TestCase;
class FlattenNestedBoolTest extends TestCase {
private $optimizer;
private $simplifier;
protected function setUp(): void {
parent::setUp();
$this->optimizer = new FlattenNestedBool();
$this->simplifier = new FlattenSingleArgumentBinaryOperation();
}
public function testOrs() {
$operator = new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_OR,
[
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "foo"),
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "bar"),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "asd"),
])
]
);
$this->assertEquals('(path eq "foo" or (path eq "bar" or path eq "asd"))', $operator->__toString());
$this->optimizer->processOperator($operator);
$this->simplifier->processOperator($operator);
$this->assertEquals('(path eq "foo" or path eq "bar" or path eq "asd")', $operator->__toString());
}
}

133
tests/lib/Files/Search/QueryOptimizer/MergeDistributiveOperationsTest.php Normal file
View File

@ -0,0 +1,133 @@
<?php
namespace Test\Files\Search\QueryOptimizer;
use OC\Files\Search\QueryOptimizer\FlattenSingleArgumentBinaryOperation;
use OC\Files\Search\QueryOptimizer\MergeDistributiveOperations;
use OC\Files\Search\SearchBinaryOperator;
use OC\Files\Search\SearchComparison;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchComparison;
use Test\TestCase;
class MergeDistributiveOperationsTest extends TestCase {
private $optimizer;
private $simplifier;
protected function setUp(): void {
parent::setUp();
$this->optimizer = new MergeDistributiveOperations();
$this->simplifier = new FlattenSingleArgumentBinaryOperation();
}
public function testBasicOrOfAnds() {
$operator = new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_OR,
[
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "foo"),
]),
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "bar"),
]),
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "asd"),
])
]
);
$this->assertEquals('((storage eq 1 and path eq "foo") or (storage eq 1 and path eq "bar") or (storage eq 1 and path eq "asd"))', $operator->__toString());
$this->optimizer->processOperator($operator);
$this->simplifier->processOperator($operator);
$this->assertEquals('(storage eq 1 and (path eq "foo" or path eq "bar" or path eq "asd"))', $operator->__toString());
}
public function testDontTouchIfNotSame() {
$operator = new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_OR,
[
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "foo"),
]),
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 2),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "bar"),
]),
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 3),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "asd"),
])
]
);
$this->assertEquals('((storage eq 1 and path eq "foo") or (storage eq 2 and path eq "bar") or (storage eq 3 and path eq "asd"))', $operator->__toString());
$this->optimizer->processOperator($operator);
$this->simplifier->processOperator($operator);
$this->assertEquals('((storage eq 1 and path eq "foo") or (storage eq 2 and path eq "bar") or (storage eq 3 and path eq "asd"))', $operator->__toString());
}
public function testMergePartial() {
$operator = new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_OR,
[
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "foo"),
]),
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "bar"),
]),
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 2),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "asd"),
])
]
);
$this->assertEquals('((storage eq 1 and path eq "foo") or (storage eq 1 and path eq "bar") or (storage eq 2 and path eq "asd"))', $operator->__toString());
$this->optimizer->processOperator($operator);
$this->simplifier->processOperator($operator);
$this->assertEquals('((storage eq 1 and (path eq "foo" or path eq "bar")) or (storage eq 2 and path eq "asd"))', $operator->__toString());
}
public function testOptimizeInside() {
$operator = new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_AND,
[
new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_OR,
[
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "foo"),
]),
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "bar"),
]),
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "storage", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "asd"),
])
]
),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "mimetype", "text")
]
);
$this->assertEquals('(((storage eq 1 and path eq "foo") or (storage eq 1 and path eq "bar") or (storage eq 1 and path eq "asd")) and mimetype eq "text")', $operator->__toString());
$this->optimizer->processOperator($operator);
$this->simplifier->processOperator($operator);
$this->assertEquals('((storage eq 1 and (path eq "foo" or path eq "bar" or path eq "asd")) and mimetype eq "text")', $operator->__toString());
}
}

120
tests/lib/Files/Search/QueryOptimizer/OrEqualsToInTest.php Normal file
View File

@ -0,0 +1,120 @@
<?php
namespace Test\Files\Search\QueryOptimizer;
use OC\Files\Search\QueryOptimizer\FlattenSingleArgumentBinaryOperation;
use OC\Files\Search\QueryOptimizer\OrEqualsToIn;
use OC\Files\Search\SearchBinaryOperator;
use OC\Files\Search\SearchComparison;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchComparison;
use Test\TestCase;
class OrEqualsToInTest extends TestCase {
private $optimizer;
private $simplifier;
protected function setUp(): void {
parent::setUp();
$this->optimizer = new OrEqualsToIn();
$this->simplifier = new FlattenSingleArgumentBinaryOperation();
}
public function testOrs() {
$operator = new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_OR,
[
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "foo"),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "bar"),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "asd"),
]
);
$this->assertEquals('(path eq "foo" or path eq "bar" or path eq "asd")', $operator->__toString());
$this->optimizer->processOperator($operator);
$this->simplifier->processOperator($operator);
$this->assertEquals('path in ["foo","bar","asd"]', $operator->__toString());
}
public function testOrsMultipleFields() {
$operator = new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_OR,
[
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "foo"),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "bar"),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "fileid", 1),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "fileid", 2),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "mimetype", "asd"),
]
);
$this->assertEquals('(path eq "foo" or path eq "bar" or fileid eq 1 or fileid eq 2 or mimetype eq "asd")', $operator->__toString());
$this->optimizer->processOperator($operator);
$this->simplifier->processOperator($operator);
$this->assertEquals('(path in ["foo","bar"] or fileid in [1,2] or mimetype eq "asd")', $operator->__toString());
}
public function testPreserveHints() {
$operator = new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_OR,
[
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "foo"),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "bar"),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "asd"),
]
);
foreach ($operator->getArguments() as $argument) {
$argument->setQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, false);
}
$this->assertEquals('(path eq "foo" or path eq "bar" or path eq "asd")', $operator->__toString());
$this->optimizer->processOperator($operator);
$this->simplifier->processOperator($operator);
$this->assertEquals('path in ["foo","bar","asd"]', $operator->__toString());
$this->assertEquals(false, $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true));
}
public function testOrSomeEq() {
$operator = new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_OR,
[
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "foo"),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "bar"),
new SearchComparison(ISearchComparison::COMPARE_LIKE, "path", "asd%"),
]
);
$this->assertEquals('(path eq "foo" or path eq "bar" or path like "asd%")', $operator->__toString());
$this->optimizer->processOperator($operator);
$this->simplifier->processOperator($operator);
$this->assertEquals('(path in ["foo","bar"] or path like "asd%")', $operator->__toString());
}
public function testOrsInside() {
$operator = new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_AND,
[
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "mimetype", "text"),
new SearchBinaryOperator(
ISearchBinaryOperator::OPERATOR_OR,
[
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "foo"),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "bar"),
new SearchComparison(ISearchComparison::COMPARE_EQUAL, "path", "asd"),
]
)
]
);
$this->assertEquals('(mimetype eq "text" and (path eq "foo" or path eq "bar" or path eq "asd"))', $operator->__toString());
$this->optimizer->processOperator($operator);
$this->simplifier->processOperator($operator);
$this->assertEquals('(mimetype eq "text" and path in ["foo","bar","asd"])', $operator->__toString());
}
}