nextcloud
/
server
mirror of https://github.com/nextcloud/server
0
0
Fork 0
Code Issues Releases Wiki Activity

chore(tests): Migrate object storage unit tests from drone to GitHub Actions 

Signed-off-by: Ferdinand Thiessen <opensource@fthiessen.de>
This commit is contained in:
Ferdinand Thiessen 2023-10-29 01:16:46 +02:00 committed by Joas Schilling
parent fceb781058
commit 8e02014b55
No known key found for this signature in database
GPG Key ID: 74434EFE0D2E2205
7 changed files with 440 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
.drone.yml
View File

@ -216,81 +216,6 @@ trigger:
- pull_request
- push
---
kind: pipeline
name: object-store-s3
steps:
- name: minio
image: ghcr.io/nextcloud/continuous-integration-minio:latest
detach: true
commands:
- mkdir /s3data
- minio server /s3data
environment:
MINIO_ROOT_USER: nextcloud
MINIO_ROOT_PASSWORD: nextcloud
- name: submodules
image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest
commands:
- git submodule update --init
- name: object-store
image: ghcr.io/nextcloud/continuous-integration-php8.0:latest
environment:
OBJECT_STORE: s3
CODECOV_TOKEN:
from_secret: CODECOV_TOKEN
commands:
- phpenmod xdebug
- ./tests/drone-wait-objectstore.sh
- TEST_SELECTION=PRIMARY-s3 ./autotest.sh sqlite
- wget https://codecov.io/bash -O codecov.sh
- bash codecov.sh -C $DRONE_COMMIT -f tests/autotest-clover-sqlite.xml
trigger:
branch:
- master
- stable*
event:
- pull_request
- push
---
kind: pipeline
name: object-store-azure
steps:
- name: submodules
image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest
commands:
- git submodule update --init
- name: object-store
image: ghcr.io/nextcloud/continuous-integration-php8.0:latest
environment:
OBJECT_STORE: azure
CODECOV_TOKEN:
from_secret: CODECOV_TOKEN
commands:
- phpenmod xdebug
- ./tests/drone-wait-objectstore.sh
- TEST_SELECTION=PRIMARY-azure ./autotest.sh sqlite
- wget https://codecov.io/bash -O codecov.sh
- bash codecov.sh -C $DRONE_COMMIT -f tests/autotest-clover-sqlite.xml
services:
- name: azurite
image: ghcr.io/nextcloud/continuous-integration-azurite:latest
environment:
executable: blob
trigger:
branch:
- master
- stable*
event:
- pull_request
- push
---
kind: pipeline
name: memcache-memcached

133
.github/workflows/object-storage-azure.yml vendored Normal file
View File

@ -0,0 +1,133 @@
name: Object storage azure
on:
pull_request:
paths:
- '.github/workflows/**'
- '3rdparty/**'
- '**/*.php'
- '**/lib/**'
- '**/tests/**'
- '**/vendor-bin/**'
- '.php-cs-fixer.dist.php'
- 'composer.json'
- 'composer.lock'
concurrency:
group: object-storage-azure-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
changes:
runs-on: ubuntu-latest
outputs:
src: ${{ steps.changes.outputs.src}}
steps:
- uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
id: changes
continue-on-error: true
with:
filters: |
src:
- '.github/workflows/**'
- '3rdparty/**'
- '**/appinfo/**'
- '**/lib/**'
- '**/templates/**'
- '**/tests/**'
- 'vendor/**'
- 'vendor-bin/**'
- '.php-cs-fixer.dist.php'
- 'composer.json'
- 'composer.lock'
- '**.php'
azure-primary-tests:
runs-on: ubuntu-latest
needs: changes
if: ${{ github.repository_owner != 'nextcloud-gmbh' && needs.changes.outputs.src != 'false' }}
strategy:
matrix:
php-versions: ['8.0', '8.1', '8.2', '8.3']
include:
- php-versions: '8.3'
coverage: true
name: php${{ matrix.php-versions }}-azure
services:
azurite:
image: mcr.microsoft.com/azure-storage/azurite
env:
AZURITE_ACCOUNTS: nextcloud:bmV4dGNsb3Vk
ports:
- 10000:10000
options: --health-cmd="nc 127.0.0.1 10000 -z" --health-interval=1s --health-retries=30
cache:
image: ghcr.io/nextcloud/continuous-integration-redis:latest
ports:
- 6379:6379/tcp
options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3
steps:
- name: Checkout server
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
submodules: true
- name: Set up php ${{ matrix.php-versions }}
uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2
with:
php-version: ${{ matrix.php-versions }}
# https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite
coverage: ${{ matrix.coverage && 'xdebug' || 'none' }}
ini-file: development
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Nextcloud
env:
OBJECT_STORE: azure
OBJECT_STORE_KEY: nextcloud
OBJECT_STORE_SECRET: bmV4dGNsb3Vk
run: |
composer install
cp tests/redis.config.php config/
cp tests/preseed-config.php config/config.php
./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0
- name: PHPUnit
env:
OBJECT_STORE: azure
OBJECT_STORE_KEY: nextcloud
OBJECT_STORE_SECRET: bmV4dGNsb3Vk
run: composer run test -- --group PRIMARY-azure ${{ matrix.coverage && ' --coverage-clover ./clover.xml' || '' }}
- name: Upload code coverage
if: ${{ !cancelled() && matrix.coverage }}
uses: codecov/codecov-action@v3
with:
files: ./clover.xml
flags: phpunit-azure
- name: Azurite logs
if: always()
run: |
docker ps -a
docker ps -aq | while read container ; do IMAGE=$(docker inspect --format='{{.Config.Image}}' $container); echo $IMAGE; docker logs $container; echo "\n\n" ; done
azure-primary-summary:
runs-on: ubuntu-latest
needs: [changes, azure-primary-tests]
if: always()
steps:
- name: Summary status
run: if ${{ needs.changes.outputs.src != 'false' && needs.azure-primary-tests.result != 'success' }}; then exit 1; fi

139
.github/workflows/object-storage-s3.yml vendored Normal file
View File

@ -0,0 +1,139 @@
name: Object storage S3
on:
pull_request:
paths:
- '.github/workflows/**'
- '3rdparty/**'
- '**/*.php'
- '**/lib/**'
- '**/tests/**'
- '**/vendor-bin/**'
- '.php-cs-fixer.dist.php'
- 'composer.json'
- 'composer.lock'
concurrency:
group: object-storage-s3-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
changes:
runs-on: ubuntu-latest
outputs:
src: ${{ steps.changes.outputs.src}}
steps:
- uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
id: changes
continue-on-error: true
with:
filters: |
src:
- '.github/workflows/**'
- '3rdparty/**'
- '**/appinfo/**'
- '**/lib/**'
- '**/templates/**'
- '**/tests/**'
- 'vendor/**'
- 'vendor-bin/**'
- '.php-cs-fixer.dist.php'
- 'composer.json'
- 'composer.lock'
- '**.php'
s3-primary-tests-minio:
runs-on: ubuntu-latest
needs: changes
if: ${{ github.repository_owner != 'nextcloud-gmbh' && needs.changes.outputs.src != 'false' }}
strategy:
matrix:
php-versions: ['8.0', '8.1', '8.2', '8.3']
include:
- php-versions: '8.3'
coverage: true
name: php${{ matrix.php-versions }}-s3
services:
cache:
image: ghcr.io/nextcloud/continuous-integration-redis:latest
ports:
- 6379:6379/tcp
options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3
minio:
image: bitnami/minio
env:
MINIO_ROOT_USER: nextcloud
MINIO_ROOT_PASSWORD: bWluaW8tc2VjcmV0LWtleS1uZXh0Y2xvdWQ=
MINIO_DEFAULT_BUCKETS: nextcloud
ports:
- "9000:9000"
steps:
- name: Checkout server
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
submodules: true
- name: Set up php ${{ matrix.php-versions }}
uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2
with:
php-version: ${{ matrix.php-versions }}
# https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite
coverage: ${{ matrix.coverage && 'xdebug' || 'none' }}
ini-file: development
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Nextcloud
env:
OBJECT_STORE: s3
OBJECT_STORE_KEY: nextcloud
OBJECT_STORE_SECRET: bWluaW8tc2VjcmV0LWtleS1uZXh0Y2xvdWQ=
run: |
composer install
cp tests/redis.config.php config/
cp tests/preseed-config.php config/config.php
./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0
- name: Wait for S3
run: |
sleep 10
curl -f -m 1 --retry-connrefused --retry 10 --retry-delay 10 http://localhost:9000/minio/health/ready
- name: PHPUnit
env:
OBJECT_STORE: s3
OBJECT_STORE_KEY: nextcloud
OBJECT_STORE_SECRET: bWluaW8tc2VjcmV0LWtleS1uZXh0Y2xvdWQ=
run: composer run test -- --group PRIMARY-s3 ${{ matrix.coverage && ' --coverage-clover ./clover.xml' || '' }}
- name: Upload code coverage
if: ${{ !cancelled() && matrix.coverage }}
uses: codecov/codecov-action@v3
with:
files: ./clover.xml
flags: phpunit-s3
- name: S3 logs
if: always()
run: |
docker ps -a
docker ps -aq | while read container ; do IMAGE=$(docker inspect --format='{{.Config.Image}}' $container); echo $IMAGE; docker logs $container; echo "\n\n" ; done
s3-primary-summary:
runs-on: ubuntu-latest
needs: [changes,s3-primary-tests-minio]
if: always()
steps:
- name: Summary status
run: if ${{ needs.changes.outputs.src != 'false' && needs.s3-primary-tests-minio.result != 'success' }}; then exit 1; fi

129
.github/workflows/object-storage-swift.yml vendored Normal file
View File

@ -0,0 +1,129 @@
name: Object storage Swift
on:
pull_request:
paths:
- '.github/workflows/**'
- '3rdparty/**'
- '**/*.php'
- '**/lib/**'
- '**/tests/**'
- '**/vendor-bin/**'
- '.php-cs-fixer.dist.php'
- 'composer.json'
- 'composer.lock'
concurrency:
group: object-storage-swift-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
changes:
runs-on: ubuntu-latest
outputs:
src: ${{ steps.changes.outputs.src}}
steps:
- uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
id: changes
continue-on-error: true
with:
filters: |
src:
- '.github/workflows/**'
- '3rdparty/**'
- '**/appinfo/**'
- '**/lib/**'
- '**/templates/**'
- '**/tests/**'
- 'vendor/**'
- 'vendor-bin/**'
- '.php-cs-fixer.dist.php'
- 'composer.json'
- 'composer.lock'
- '**.php'
swift-primary-tests:
runs-on: ubuntu-latest
needs: changes
if: ${{ github.repository_owner != 'nextcloud-gmbh' && needs.changes.outputs.src != 'false' }}
strategy:
matrix:
php-versions: ['8.0', '8.1', '8.2', '8.3']
include:
- php-versions: '8.3'
coverage: true
name: php${{ matrix.php-versions }}-swift
services:
cache:
image: ghcr.io/nextcloud/continuous-integration-redis:latest
ports:
- 6379:6379/tcp
options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3
swift:
image: ghcr.io/cscfi/docker-keystone-swift
ports:
- 5000:5000
- 8080:8080
steps:
- name: Checkout server
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
submodules: true
- name: Set up php ${{ matrix.php-versions }}
uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2
with:
php-version: ${{ matrix.php-versions }}
# https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite
coverage: ${{ matrix.coverage && 'xdebug' || 'none' }}
ini-file: development
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Nextcloud
env:
OBJECT_STORE: swift
OBJECT_STORE_SECRET: veryfast
run: |
composer install
cp tests/redis.config.php config/
cp tests/preseed-config.php config/config.php
./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0
- name: PHPUnit
env:
OBJECT_STORE: swift
OBJECT_STORE_SECRET: veryfast
run: composer run test -- --group PRIMARY-swift ${{ matrix.coverage && ' --coverage-clover ./clover.xml' || '' }}
- name: Upload code coverage
if: ${{ !cancelled() && matrix.coverage }}
uses: codecov/codecov-action@v3
with:
files: ./clover.xml
flags: phpunit-swift
- name: Swift logs
if: always()
run: |
docker ps -a
docker ps -aq | while read container ; do IMAGE=$(docker inspect --format='{{.Config.Image}}' $container); echo $IMAGE; docker logs $container; echo "\n\n" ; done
swift-primary-summary:
runs-on: ubuntu-latest
needs: [changes,swift-primary-tests]
if: always()
steps:
- name: Summary status
run: if ${{ needs.changes.outputs.src != 'false' && needs.swift-primary-tests.result != 'success' }}; then exit 1; fi

87
.github/workflows/s3-primary.yml vendored
View File

@ -1,87 +0,0 @@
name: S3 primary storage
on:
pull_request:
paths:
- '.github/workflows/**'
- '3rdparty/**'
- '**/*.php'
- '**/lib/**'
- '**/tests/**'
- '**/vendor-bin/**'
- '.php-cs-fixer.dist.php'
- 'composer.json'
- 'composer.lock'
concurrency:
group: s3-primary-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
s3-primary-tests-minio:
runs-on: ubuntu-20.04
if: ${{ github.repository_owner != 'nextcloud-gmbh' }}
strategy:
# do not stop on another job's failure
fail-fast: false
matrix:
php-versions: ['8.0']
key: ['objectstore', 'objectstore_multibucket']
name: php${{ matrix.php-versions }}-${{ matrix.key }}-minio
services:
minio:
env:
MINIO_ACCESS_KEY: minio
MINIO_SECRET_KEY: minio123
image: bitnami/minio:2021.12.29
ports:
- "9000:9000"
steps:
- name: Checkout server
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
submodules: true
- name: Set up php ${{ matrix.php-versions }}
uses: shivammathur/setup-php@c5fc0d8281aba02c7fda07d3a70cc5371548067d #v2.25.2
with:
php-version: ${{ matrix.php-versions }}
extensions: mbstring, fileinfo, intl, sqlite, pdo_sqlite, zip, gd
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Nextcloud
run: |
composer install
mkdir data
echo '<?php $CONFIG=["${{ matrix.key }}" => ["class" => "OC\Files\ObjectStore\S3", "arguments" => ["bucket" => "nextcloud", "autocreate" => true, "key" => "minio", "secret" => "minio123", "hostname" => "localhost", "port" => 9000, "use_ssl" => false, "use_path_style" => true, "uploadPartSize" => 52428800]]];' > config/config.php
./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
php -f index.php
- name: Wait for S3
run: |
sleep 10
curl -f -m 1 --retry-connrefused --retry 10 --retry-delay 10 http://localhost:9000/minio/health/ready
- name: PHPUnit
run: composer run test:db
- name: S3 logs
if: always()
run: |
docker ps -a
docker logs $(docker ps -aq)
s3-primary-summary:
runs-on: ubuntu-latest
needs: [s3-primary-tests-minio]
if: always()
steps:
- name: Summary status
run: if ${{ needs.s3-primary-tests-minio.result != 'success' }}; then exit 1; fi

4
tests/lib/Files/ObjectStore/SwiftTest.php
View File

@ -38,4 +38,8 @@ class SwiftTest extends ObjectStoreTest {
return new Swift($config['arguments']);
}
public function testFseekSize() {
$this->markTestSkipped('Swift does not support seeking at the moment');
}
}

89
tests/preseed-config.php
View File

@ -25,71 +25,52 @@ if (getenv('OBJECT_STORE') === 's3') {
'arguments' => [
'bucket' => 'nextcloud',
'autocreate' => true,
'key' => 'nextcloud',
'secret' => 'nextcloud',
'hostname' => getenv('DRONE') === 'true' ? 'minio' : 'localhost',
'key' => getenv('OBJECT_STORE_KEY') ?: 'nextcloud',
'secret' => getenv('OBJECT_STORE_SECRET') ?: 'nextcloud',
'hostname' => getenv('OBJECT_STORE_HOST') ?: 'localhost',
'port' => 9000,
'use_ssl' => false,
// required for some non amazon s3 implementations
'use_path_style' => true
]
];
}
if (getenv('OBJECT_STORE') === 'swift') {
$swiftHost = getenv('DRONE') === 'true' ? 'dockswift' : 'localhost';
if (getenv('SWIFT-AUTH') === 'v2.0') {
$CONFIG['objectstore'] = [
'class' => 'OC\\Files\\ObjectStore\\Swift',
'arguments' => [
'autocreate' => true,
'username' => 'swift',
'tenantName' => 'service',
'password' => 'swift',
'serviceName' => 'swift',
'region' => 'regionOne',
'url' => "http://$swiftHost:5000/v2.0",
'bucket' => 'nextcloud'
]
];
} else {
$CONFIG['objectstore'] = [
'class' => 'OC\\Files\\ObjectStore\\Swift',
'arguments' => [
'autocreate' => true,
'user' => [
'name' => 'swift',
'password' => 'swift',
'domain' => [
'name' => 'default',
]
],
'scope' => [
'project' => [
'name' => 'service',
'domain' => [
'name' => 'default',
],
],
],
'tenantName' => 'service',
'serviceName' => 'swift',
'region' => 'regionOne',
'url' => "http://$swiftHost:5000/v3",
'bucket' => 'nextcloud'
]
];
}
}
if (getenv('OBJECT_STORE') === 'azure') {
} elseif (getenv('OBJECT_STORE') === 'azure') {
$CONFIG['objectstore'] = [
'class' => 'OC\\Files\\ObjectStore\\Azure',
'arguments' => [
'container' => 'test',
'account_name' => 'devstoreaccount1',
'account_key' => 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==',
'endpoint' => 'http://' . (getenv('DRONE') === 'true' ? 'azurite' : 'localhost') . ':10000/devstoreaccount1',
'account_name' => getenv('OBJECT_STORE_KEY') ?: 'devstoreaccount1',
'account_key' => getenv('OBJECT_STORE_SECRET') ?: 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==',
'endpoint' => 'http://' . (getenv('OBJECT_STORE_HOST') ?: 'localhost') . ':10000/' . (getenv('OBJECT_STORE_KEY') ?: 'devstoreaccount1'),
'autocreate' => true
]
];
} elseif (getenv('OBJECT_STORE') === 'swift') {
$swiftHost = getenv('OBJECT_STORE_HOST') ?: 'localhost:5000';
$CONFIG['objectstore'] = [
'class' => 'OC\\Files\\ObjectStore\\Swift',
'arguments' => [
'autocreate' => true,
'user' => [
'name' => getenv('OBJECT_STORE_KEY') ?: 'swift',
'password' => getenv('OBJECT_STORE_SECRET') ?: 'swift',
'domain' => [
'name' => 'Default',
],
],
'scope' => [
'project' => [
'name' => 'service',
'domain' => [
'name' => 'Default',
],
],
],
'serviceName' => 'service',
'region' => 'RegionOne',
'url' => "http://$swiftHost/v3",
'bucket' => 'nextcloud',
]
];
}