nextcloud
/
server
mirror of https://github.com/nextcloud/server
0
0
Fork 0
Code Issues Releases Wiki Activity

Change X-Robots-Tag header from "none" to "noindex, nofollow" 

While "none" is indeed equivalent to "noindex, nofollow" for Google, but seems to be not supported by Bing and probably other search engines.

https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta/name#other_metadata_names
https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag?hl=de#comma-separated-list
https://www.bing.com/webmasters/help/which-robots-metatags-does-bing-support-5198d240

Signed-off-by: MichaIng <micha@dietpi.com>
This commit is contained in:
MichaIng 2023-02-13 14:09:13 +01:00
parent b36a31c918
commit 5f90b8eb11
No known key found for this signature in database
GPG Key ID: 0442B9ADE65643FE
12 changed files with 80 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.htaccess
View File

@ -31,7 +31,7 @@
Header always set X-Permitted-Cross-Domain-Policies "none"
Header onsuccess unset X-Robots-Tag
Header always set X-Robots-Tag "none"
Header always set X-Robots-Tag "noindex, nofollow"
Header onsuccess unset X-XSS-Protection
Header always set X-XSS-Protection "1; mode=block"

4
build/integration/features/carddav.feature
View File

@ -46,7 +46,7 @@ Feature: carddav
|X-Content-Type-Options |nosniff|
|X-Frame-Options|SAMEORIGIN|
|X-Permitted-Cross-Domain-Policies|none|
|X-Robots-Tag|none|
|X-Robots-Tag|noindex, nofollow|
|X-XSS-Protection|1; mode=block|
Scenario: Exporting the picture of ones own contact
@ -60,5 +60,5 @@ Feature: carddav
|X-Content-Type-Options |nosniff|
|X-Frame-Options|SAMEORIGIN|
|X-Permitted-Cross-Domain-Policies|none|
|X-Robots-Tag|none|
|X-Robots-Tag|noindex, nofollow|
|X-XSS-Protection|1; mode=block|

2
build/integration/features/dav-v2.feature
View File

@ -27,7 +27,7 @@ Feature: dav-v2
|X-Content-Type-Options |nosniff|
|X-Frame-Options|SAMEORIGIN|
|X-Permitted-Cross-Domain-Policies|none|
|X-Robots-Tag|none|
|X-Robots-Tag|noindex, nofollow|
|X-XSS-Protection|1; mode=block|
And Downloaded content should start with "Welcome to your Nextcloud account!"

4
build/integration/features/webdav-related.feature
View File

@ -53,7 +53,7 @@ Feature: webdav-related
And User "user0" moves file "/textfile0.txt" to "/testshare/textfile0.txt"
And the HTTP status code should be "403"
When Downloading file "/testshare/textfile0.txt"
Then the HTTP status code should be "404"
Then the HTTP status code should be "404"
Scenario: Moving a file to overwrite a file in a folder with no permissions
Given using old dav path
@ -251,7 +251,7 @@ Feature: webdav-related
|X-Content-Type-Options |nosniff|
|X-Frame-Options|SAMEORIGIN|
|X-Permitted-Cross-Domain-Policies|none|
|X-Robots-Tag|none|
|X-Robots-Tag|noindex, nofollow|
|X-XSS-Protection|1; mode=block|
And Downloaded content should start with "Welcome to your Nextcloud account!"

4
core/js/setupchecks.js
View File

@ -628,13 +628,13 @@
if (xhr.status === 200) {
var securityHeaders = {
'X-Content-Type-Options': ['nosniff'],
'X-Robots-Tag': ['none'],
'X-Robots-Tag': ['noindex, nofollow'],
'X-Frame-Options': ['SAMEORIGIN', 'DENY'],
'X-Permitted-Cross-Domain-Policies': ['none'],
};
for (var header in securityHeaders) {
var option = securityHeaders[header][0];
if(!xhr.getResponseHeader(header) || xhr.getResponseHeader(header).toLowerCase() !== option.toLowerCase()) {
if(!xhr.getResponseHeader(header) || xhr.getResponseHeader(header).replace(/, /, ',').toLowerCase() !== option.replace(/, /, ',').toLowerCase()) {
var msg = t('core', 'The "{header}" HTTP header is not set to "{expected}". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.', {header: header, expected: option});
if(xhr.getResponseHeader(header) && securityHeaders[header].length > 1 && xhr.getResponseHeader(header).toLowerCase() === securityHeaders[header][1].toLowerCase()) {
msg = t('core', 'The "{header}" HTTP header is not set to "{expected}". Some features might not work correctly, as it is recommended to adjust this setting accordingly.', {header: header, expected: option});

89
core/js/tests/specs/setupchecksSpec.js
View File

@ -1569,7 +1569,7 @@ describe('OC.SetupChecks tests', function() {
msg: 'The "X-Content-Type-Options" HTTP header is not set to "nosniff". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
type: OC.SetupChecks.MESSAGE_TYPE_WARNING
}, {
msg: 'The "X-Robots-Tag" HTTP header is not set to "none". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
msg: 'The "X-Robots-Tag" HTTP header is not set to "noindex, nofollow". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
type: OC.SetupChecks.MESSAGE_TYPE_WARNING
}, {
msg: 'The "X-Frame-Options" HTTP header is not set to "SAMEORIGIN". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
@ -1596,7 +1596,7 @@ describe('OC.SetupChecks tests', function() {
suite.server.requests[0].respond(
200,
{
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'Strict-Transport-Security': 'max-age=15768000;preload',
'X-Permitted-Cross-Domain-Policies': 'none',
@ -1627,7 +1627,7 @@ describe('OC.SetupChecks tests', function() {
{
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'Strict-Transport-Security': 'max-age=15768000',
'X-Permitted-Cross-Domain-Policies': 'none',
@ -1641,6 +1641,49 @@ describe('OC.SetupChecks tests', function() {
});
});
describe('check X-Robots-Tag header', function() {
it('should return no message if X-Robots-Tag is set to noindex,nofollow without space', function(done) {
protocolStub.returns('https');
var result = OC.SetupChecks.checkGeneric();
suite.server.requests[0].respond(200, {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'noindex,nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
});
result.done(function( data, s, x ){
expect(data).toEqual([]);
done();
});
});
it('should return a message if X-Robots-Tag is set to none', function(done) {
protocolStub.returns('https');
var result = OC.SetupChecks.checkGeneric();
suite.server.requests[0].respond(200, {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
});
result.done(function( data, s, x ){
expect(data).toEqual([
{
msg: 'The "X-Robots-Tag" HTTP header is not set to "noindex, nofollow". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
type: OC.SetupChecks.MESSAGE_TYPE_WARNING
}
]);
done();
});
});
});
describe('check X-XSS-Protection header', function() {
it('should return no message if X-XSS-Protection is set to 1; mode=block; report=https://example.com', function(done) {
protocolStub.returns('https');
@ -1650,7 +1693,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block; report=https://example.com',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1670,7 +1713,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1690,7 +1733,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1715,7 +1758,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '0',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1742,7 +1785,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1762,7 +1805,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer-when-downgrade',
@ -1782,7 +1825,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'strict-origin',
@ -1802,7 +1845,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'strict-origin-when-cross-origin',
@ -1822,7 +1865,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'same-origin',
@ -1842,7 +1885,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'origin',
@ -1867,7 +1910,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'origin-when-cross-origin',
@ -1892,7 +1935,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'unsafe-url',
@ -1919,7 +1962,7 @@ describe('OC.SetupChecks tests', function() {
{
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1965,7 +2008,7 @@ describe('OC.SetupChecks tests', function() {
{
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1990,7 +2033,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15551999',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -2015,7 +2058,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'iAmABogusHeader342',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -2039,7 +2082,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -2059,7 +2102,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=99999999',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -2079,7 +2122,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=99999999; includeSubDomains',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -2099,7 +2142,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=99999999; preload; includeSubDomains',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',

2
lib/private/legacy/OC_Response.php
View File

@ -99,7 +99,7 @@ class OC_Response {
header('X-Content-Type-Options: nosniff'); // Disable sniffing the content type for IE
header('X-Frame-Options: SAMEORIGIN'); // Disallow iFraming from other domains
header('X-Permitted-Cross-Domain-Policies: none'); // https://www.adobe.com/devnet/adobe-media-server/articles/cross-domain-xml-for-streaming.html
header('X-Robots-Tag: none'); // https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag
header('X-Robots-Tag: noindex, nofollow'); // https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag
header('X-XSS-Protection: 1; mode=block'); // Enforce browser based XSS filters
}
}

2
lib/public/AppFramework/Http/Response.php
View File

@ -257,7 +257,7 @@ class Response {
$this->headers['Content-Security-Policy'] = $this->getContentSecurityPolicy()->buildPolicy();
$this->headers['Feature-Policy'] = $this->getFeaturePolicy()->buildPolicy();
$this->headers['X-Robots-Tag'] = 'none';
$this->headers['X-Robots-Tag'] = 'noindex, nofollow';
if ($this->ETag) {
$mergeWith['ETag'] = '"' . $this->ETag . '"';

2
tests/data/setUploadLimit/htaccess
View File

@ -11,7 +11,7 @@
# Add security and privacy related headers
Header set X-Content-Type-Options "nosniff"
Header set X-XSS-Protection "1; mode=block"
Header set X-Robots-Tag "none"
Header set X-Robots-Tag "noindex, nofollow"
Header set X-Frame-Options "SAMEORIGIN"
SetEnv modHeadersAvailable true
</IfModule>

2
tests/lib/AppFramework/Controller/ControllerTest.php
View File

@ -116,7 +116,7 @@ class ControllerTest extends \Test\TestCase {
'Content-Security-Policy' => "default-src 'none';base-uri 'none';manifest-src 'self';frame-ancestors 'none'",
'Feature-Policy' => "autoplay 'none';camera 'none';fullscreen 'none';geolocation 'none';microphone 'none';payment 'none'",
'X-Request-Id' => $this->request->getId(),
'X-Robots-Tag' => 'none',
'X-Robots-Tag' => 'noindex, nofollow',
];
$response = $this->controller->customDataResponse(['hi']);

2
tests/lib/AppFramework/Http/DataResponseTest.php
View File

@ -67,7 +67,7 @@ class DataResponseTest extends \Test\TestCase {
'Cache-Control' => 'no-cache, no-store, must-revalidate',
'Content-Security-Policy' => "default-src 'none';base-uri 'none';manifest-src 'self';frame-ancestors 'none'",
'Feature-Policy' => "autoplay 'none';camera 'none';fullscreen 'none';geolocation 'none';microphone 'none';payment 'none'",
'X-Robots-Tag' => 'none',
'X-Robots-Tag' => 'noindex, nofollow',
'X-Request-Id' => \OC::$server->get(IRequest::class)->getId(),
];
$expectedHeaders = array_merge($expectedHeaders, $headers);

2
tests/lib/AppFramework/Http/ResponseTest.php
View File

@ -51,7 +51,7 @@ class ResponseTest extends \Test\TestCase {
'Last-Modified' => 1,
'ETag' => 3,
'Something-Else' => 'hi',
'X-Robots-Tag' => 'none',
'X-Robots-Tag' => 'noindex, nofollow',
];
$this->childResponse->setHeaders($expected);