mirror of https://github.com/nextcloud/desktop
Remove qtokenizer in favour of Qt6 QStringTokenizer
Signed-off-by: Claudio Cambra <claudio.cambra@nextcloud.com>
This commit is contained in:
Claudio Cambra
Matthieu Gallien
parent
29b0d2b8ad
commit
9932956686
|
|
@ -1,261 +0,0 @@
|
|||
/****************************************************************************
|
||||
**
|
||||
** Copyright (C) 2014 Daniel Molkentin <daniel@molkentin.de>
|
||||
** Contact: http://www.qt-project.org/legal
|
||||
**
|
||||
** This file is part of the QtNetwork module of the Qt Toolkit.
|
||||
**
|
||||
** $QT_BEGIN_LICENSE:LGPL$
|
||||
** Commercial License Usage
|
||||
** Licensees holding valid commercial Qt licenses may use this file in
|
||||
** accordance with the commercial license agreement provided with the
|
||||
** Software or, alternatively, in accordance with the terms contained in
|
||||
** a written agreement between you and Digia. For licensing terms and
|
||||
** conditions see http://qt.digia.com/licensing. For further information
|
||||
** use the contact form at http://qt.digia.com/contact-us.
|
||||
**
|
||||
** GNU Lesser General Public License Usage
|
||||
** Alternatively, this file may be used under the terms of the GNU Lesser
|
||||
** General Public License version 2.1 as published by the Free Software
|
||||
** Foundation and appearing in the file LICENSE.LGPL included in the
|
||||
** packaging of this file. Please review the following information to
|
||||
** ensure the GNU Lesser General Public License version 2.1 requirements
|
||||
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
|
||||
**
|
||||
** In addition, as a special exception, Digia gives you certain additional
|
||||
** rights. These rights are described in the Digia Qt LGPL Exception
|
||||
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
|
||||
**
|
||||
** GNU General Public License Usage
|
||||
** Alternatively, this file may be used under the terms of the GNU
|
||||
** General Public License version 3.0 as published by the Free Software
|
||||
** Foundation and appearing in the file LICENSE.GPL included in the
|
||||
** packaging of this file. Please review the following information to
|
||||
** ensure the GNU General Public License version 3.0 requirements will be
|
||||
** met: http://www.gnu.org/copyleft/gpl.html.
|
||||
**
|
||||
**
|
||||
** $QT_END_LICENSE$
|
||||
**
|
||||
****************************************************************************/
|
||||
|
||||
#ifndef TOKENIZER_H
|
||||
#define TOKENIZER_H
|
||||
|
||||
#include <QString>
|
||||
#include <QByteArray>
|
||||
#include <QSharedPointer>
|
||||
|
||||
QT_BEGIN_NAMESPACE
|
||||
|
||||
template <class T, class const_iterator>
|
||||
struct QTokenizerPrivate {
|
||||
using char_type = typename T::value_type;
|
||||
|
||||
struct State {
|
||||
bool inQuote = false;
|
||||
bool inEscape = false;
|
||||
char_type quoteChar = '\0';
|
||||
};
|
||||
|
||||
QTokenizerPrivate(const T& _string, const T& _delims) :
|
||||
string(_string)
|
||||
, begin(string.begin())
|
||||
, end(string.end())
|
||||
, tokenBegin(end)
|
||||
, tokenEnd(begin)
|
||||
, delimiters(_delims)
|
||||
{
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isDelimiter(char_type c) const {
|
||||
return delimiters.contains(c);
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isQuote(char_type c) const {
|
||||
return quotes.contains(c);
|
||||
}
|
||||
|
||||
// Returns true if a delimiter was not hit
|
||||
bool nextChar(State* state, char_type c) {
|
||||
if (state->inQuote) {
|
||||
if (state->inEscape) {
|
||||
state->inEscape = false;
|
||||
} else if (c == '\\') {
|
||||
state->inEscape = true;
|
||||
} else if (c == state->quoteChar) {
|
||||
state->inQuote = false;
|
||||
}
|
||||
} else {
|
||||
if (isDelimiter(c))
|
||||
return false;
|
||||
state->inQuote = isQuote(state->quoteChar = c);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
T string;
|
||||
// ### copies begin and end for performance, premature optimization?
|
||||
const_iterator begin;
|
||||
const_iterator end;
|
||||
const_iterator tokenBegin;
|
||||
const_iterator tokenEnd;
|
||||
T delimiters;
|
||||
T quotes;
|
||||
bool isDelim = false;
|
||||
bool returnDelimiters = false;
|
||||
bool returnQuotes = false;
|
||||
};
|
||||
|
||||
template <class T, class const_iterator = typename T::const_iterator>
|
||||
class QTokenizer {
|
||||
public:
|
||||
using char_type = typename T::value_type;
|
||||
|
||||
/*!
|
||||
\class QTokenizer
|
||||
\inmodule QtNetwork
|
||||
\brief QTokenizer tokenizes Strings on QString, QByteArray,
|
||||
std::string or std::wstring
|
||||
|
||||
Example Usage:
|
||||
|
||||
\code
|
||||
QString str = ...;
|
||||
QByteArrayTokenizer tokenizer(str, "; ");
|
||||
tokenizer.setQuoteCharacters("\"'");
|
||||
tokenizer.setReturnDelimiters(true);
|
||||
while (tokenizer.hasNext()) {
|
||||
QByteArray token = tokenizer.next();
|
||||
bool isDelimiter = tokenizer.isDelimiter();
|
||||
...
|
||||
}
|
||||
\endcode
|
||||
|
||||
\param string The string to tokenize
|
||||
\param delimiters A string containing delimiters
|
||||
|
||||
\sa QStringTokenizer, QByteArrayTokenizer, StringTokenizer, WStringTokenizer
|
||||
*/
|
||||
QTokenizer(const T& string, const T& delimiters)
|
||||
: d(new QTokenizerPrivate<T, const_iterator>(string, delimiters))
|
||||
{ }
|
||||
|
||||
/*!
|
||||
Whether or not to return delimiters as tokens
|
||||
\see setQuoteCharacters
|
||||
*/
|
||||
void setReturnDelimiters(bool enable) { d->returnDelimiters = enable; }
|
||||
|
||||
|
||||
/*!
|
||||
Sets characters that are considered to start and end quotes.
|
||||
|
||||
When between two characters considered a quote, delimiters will
|
||||
be ignored.
|
||||
|
||||
When between quotes, blackslash characters will cause the QTokenizer
|
||||
to skip the next character.
|
||||
|
||||
\param quotes Characters that delimit quotes.
|
||||
*/
|
||||
void setQuoteCharacters(const T& quotes) { d->quotes = quotes; }
|
||||
|
||||
|
||||
/*!
|
||||
Whether or not to return delimiters as tokens
|
||||
\see setQuoteCharacters
|
||||
*/
|
||||
void setReturnQuoteCharacters(bool enable) { d->returnQuotes = enable; }
|
||||
|
||||
|
||||
/*!
|
||||
Retrieve next token.
|
||||
|
||||
Returns true if there are more tokens, false otherwise.
|
||||
|
||||
\sa next()
|
||||
*/
|
||||
bool hasNext()
|
||||
{
|
||||
typename QTokenizerPrivate<T, const_iterator>::State state;
|
||||
d->isDelim = false;
|
||||
for (;;) {
|
||||
d->tokenBegin = d->tokenEnd;
|
||||
if (d->tokenEnd == d->end)
|
||||
return false;
|
||||
d->tokenEnd++;
|
||||
if (d->nextChar(&state, *d->tokenBegin))
|
||||
break;
|
||||
if (d->returnDelimiters) {
|
||||
d->isDelim = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
while (d->tokenEnd != d->end && d->nextChar(&state, *d->tokenEnd)) {
|
||||
d->tokenEnd++;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/*!
|
||||
Resets the tokenizer to the starting position.
|
||||
*/
|
||||
void reset() {
|
||||
d->tokenEnd = d->begin;
|
||||
}
|
||||
|
||||
/*!
|
||||
Returns true if the current token is a delimiter,
|
||||
if one more more delimiting characters have been set.
|
||||
*/
|
||||
[[nodiscard]] bool isDelimiter() const { return d->isDelim; }
|
||||
|
||||
/*!
|
||||
Returns the current token.
|
||||
|
||||
Use \c hasNext() to fetch the next token.
|
||||
*/
|
||||
[[nodiscard]] T next() const {
|
||||
int len = std::distance(d->tokenBegin, d->tokenEnd);
|
||||
const_iterator tmpStart = d->tokenBegin;
|
||||
if (!d->returnQuotes && len > 1 && d->isQuote(*d->tokenBegin)) {
|
||||
tmpStart++;
|
||||
len -= 2;
|
||||
}
|
||||
return T(tmpStart, len);
|
||||
}
|
||||
|
||||
private:
|
||||
friend class QStringTokenizer;
|
||||
QSharedPointer<QTokenizerPrivate<T, const_iterator> > d;
|
||||
};
|
||||
|
||||
class QStringTokenizer : public QTokenizer<QString> {
|
||||
public:
|
||||
QStringTokenizer(const QString &string, const QString &delim) :
|
||||
QTokenizer<QString, QString::const_iterator>(string, delim) {}
|
||||
/**
|
||||
* @brief Like \see next(), but returns a lightweight string reference
|
||||
* @return A reference to the token within the string
|
||||
*/
|
||||
QStringRef stringRef() {
|
||||
// If those differences overflow an int we'd have a veeeeeery long string in memory
|
||||
int begin = std::distance(d->begin, d->tokenBegin);
|
||||
int end = std::distance(d->tokenBegin, d->tokenEnd);
|
||||
if (!d->returnQuotes && d->isQuote(*d->tokenBegin)) {
|
||||
begin++;
|
||||
end -= 2;
|
||||
}
|
||||
return QStringRef(&d->string, begin, end);
|
||||
}
|
||||
};
|
||||
|
||||
using QByteArrayTokenizer = QTokenizer<QByteArray>;
|
||||
using StringTokenizer = QTokenizer<std::string>;
|
||||
using WStringTokenizer = QTokenizer<std::wstring>;
|
||||
|
||||
QT_END_NAMESPACE
|
||||
|
||||
#endif // TOKENIZER_H
|
||||
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
TEMPLATE = subdirs
|
||||
SUBDIRS = test
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
TEMPLATE = app
|
||||
QT += testlib
|
||||
CONFIG += testlib
|
||||
TARGET = test
|
||||
INCLUDEPATH += . ..
|
||||
|
||||
# Input
|
||||
SOURCES += tst_qtokenizer.cpp
|
||||
|
|
@ -1,139 +0,0 @@
|
|||
#include <QtTest>
|
||||
|
||||
#include "qtokenizer.h"
|
||||
|
||||
namespace {
|
||||
const QString simple = QLatin1String("A simple tokenizer test");
|
||||
const QString quoted = QLatin1String("\"Wait for me!\" he shouted");
|
||||
}
|
||||
|
||||
class TestTokenizer : public QObject
|
||||
{
|
||||
Q_OBJECT
|
||||
private slots:
|
||||
void tokenizeQStringSimple() {
|
||||
QStringTokenizer tokenizer(simple, " ");
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.next(), QLatin1String("A"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.next(), QLatin1String("simple"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.next(), QLatin1String("tokenizer"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.next(), QLatin1String("test"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), false);
|
||||
}
|
||||
|
||||
void tokenizeQStringSimpleRef() {
|
||||
QStringTokenizer tokenizer(simple, " ");
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QVERIFY(tokenizer.stringRef() == QLatin1String("A"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QVERIFY(tokenizer.stringRef() == QLatin1String("simple"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QVERIFY(tokenizer.stringRef() == QLatin1String("tokenizer"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QVERIFY(tokenizer.stringRef() == QLatin1String("test"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), false);
|
||||
}
|
||||
|
||||
void tokenizeQStringQuoted() {
|
||||
const QString multiquote(QLatin1String("\"'Billy - the Kid' is dead!\""));
|
||||
QStringTokenizer tokenizer(multiquote, " -");
|
||||
tokenizer.setQuoteCharacters("\"");
|
||||
tokenizer.setReturnQuoteCharacters(true);
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.next(), QLatin1String("\"'Billy - the Kid' is dead!\""));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), false);
|
||||
}
|
||||
|
||||
void tokenizeQStringSkipQuotes() {
|
||||
const QString multiquote(QLatin1String("\"'Billy - the Kid' is dead!\""));
|
||||
QStringTokenizer tokenizer(multiquote, " ");
|
||||
tokenizer.setQuoteCharacters("\"");
|
||||
tokenizer.setReturnQuoteCharacters(false);
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.next(), QLatin1String("'Billy - the Kid' is dead!"));
|
||||
QCOMPARE(tokenizer.stringRef().toString(), QLatin1String("'Billy - the Kid' is dead!"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), false);
|
||||
}
|
||||
|
||||
|
||||
void tokenizeQStringWithDelims() {
|
||||
const QString delims(QLatin1String("I;Insist,On/a-Delimiter"));
|
||||
QStringTokenizer tokenizer(delims, ";,/-");
|
||||
tokenizer.setReturnDelimiters(true);
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.isDelimiter(), false);
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.isDelimiter(), true);
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.isDelimiter(), false);
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.isDelimiter(), true);
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.isDelimiter(), false);
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.isDelimiter(), true);
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.isDelimiter(), false);
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.isDelimiter(), true);
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.isDelimiter(), false);
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), false);
|
||||
}
|
||||
|
||||
void resetTokenizer() {
|
||||
for (int i = 0; i < 2; i++) {
|
||||
QStringTokenizer tokenizer(simple, " ");
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.next(), QLatin1String("A"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.next(), QLatin1String("simple"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.next(), QLatin1String("tokenizer"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), true);
|
||||
QCOMPARE(tokenizer.next(), QLatin1String("test"));
|
||||
|
||||
QCOMPARE(tokenizer.hasNext(), false);
|
||||
|
||||
tokenizer.reset();
|
||||
}
|
||||
}
|
||||
|
||||
// ### QByteArray, other types
|
||||
};
|
||||
|
||||
QTEST_APPLESS_MAIN(TestTokenizer)
|
||||
|
||||
#include "tst_qtokenizer.moc"
|
||||
|
||||
|
|
@ -14,9 +14,6 @@ target_link_libraries(cmdCore
|
|||
Qt::Network
|
||||
)
|
||||
|
||||
# Need tokenizer for netrc parser
|
||||
target_include_directories(cmdCore PRIVATE ${CMAKE_SOURCE_DIR}/src/3rdparty/qtokenizer)
|
||||
|
||||
if(UNIX AND NOT APPLE)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIE")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIE")
|
||||
|
|
|
|||
|
|
@ -15,8 +15,7 @@
|
|||
#include <QDir>
|
||||
#include <QFile>
|
||||
#include <QTextStream>
|
||||
|
||||
#include <qtokenizer.h>
|
||||
#include <QStringTokenizer>
|
||||
|
||||
#include <QDebug>
|
||||
|
||||
|
|
@ -59,33 +58,32 @@ bool NetrcParser::parse()
|
|||
}
|
||||
QString content = netrc.readAll();
|
||||
|
||||
QStringTokenizer tokenizer(content, " \n\t");
|
||||
tokenizer.setQuoteCharacters("\"'");
|
||||
auto tokenizer = QStringTokenizer{content, u" \n\t"};
|
||||
|
||||
LoginPair pair;
|
||||
QString machine;
|
||||
bool isDefault = false;
|
||||
while (tokenizer.hasNext()) {
|
||||
QString key = tokenizer.next();
|
||||
for(auto itToken = tokenizer.cbegin(); itToken != tokenizer.cend(); ++itToken) {
|
||||
const auto key = *itToken;
|
||||
if (key == defaultKeyword) {
|
||||
tryAddEntryAndClear(machine, pair, isDefault);
|
||||
isDefault = true;
|
||||
continue; // don't read a value
|
||||
}
|
||||
|
||||
if (!tokenizer.hasNext()) {
|
||||
if (itToken != tokenizer.cend()) {
|
||||
qDebug() << "error fetching value for" << key;
|
||||
return false;
|
||||
}
|
||||
QString value = tokenizer.next();
|
||||
auto value = *(++itToken);
|
||||
|
||||
if (key == machineKeyword) {
|
||||
tryAddEntryAndClear(machine, pair, isDefault);
|
||||
machine = value;
|
||||
machine = value.toString();
|
||||
} else if (key == loginKeyword) {
|
||||
pair.first = value;
|
||||
pair.first = value.toString();
|
||||
} else if (key == passwordKeyword) {
|
||||
pair.second = value;
|
||||
pair.second = value.toString();
|
||||
} // ignore unsupported tokens
|
||||
}
|
||||
tryAddEntryAndClear(machine, pair, isDefault);
|
||||
|
|
|
|||
Loading…
Reference in New Issue