From 99329566866bf50c745b8a7e87b3d0a8ad843a83 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 22 Nov 2022 14:02:50 +0100 Subject: [PATCH] Remove qtokenizer in favour of Qt6 QStringTokenizer Signed-off-by: Claudio Cambra --- src/3rdparty/qtokenizer/qtokenizer.h | 261 ------------------ src/3rdparty/qtokenizer/qtokenizer.pro | 2 - src/3rdparty/qtokenizer/test/test.pro | 8 - .../qtokenizer/test/tst_qtokenizer.cpp | 139 ---------- src/cmd/CMakeLists.txt | 3 - src/cmd/netrcparser.cpp | 20 +- 6 files changed, 9 insertions(+), 424 deletions(-) delete mode 100644 src/3rdparty/qtokenizer/qtokenizer.h delete mode 100644 src/3rdparty/qtokenizer/qtokenizer.pro delete mode 100644 src/3rdparty/qtokenizer/test/test.pro delete mode 100644 src/3rdparty/qtokenizer/test/tst_qtokenizer.cpp diff --git a/src/3rdparty/qtokenizer/qtokenizer.h b/src/3rdparty/qtokenizer/qtokenizer.h deleted file mode 100644 index e1b856c3b..000000000 --- a/src/3rdparty/qtokenizer/qtokenizer.h +++ /dev/null @@ -1,261 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2014 Daniel Molkentin -** Contact: http://www.qt-project.org/legal -** -** This file is part of the QtNetwork module of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and Digia. For licensing terms and -** conditions see http://qt.digia.com/licensing. For further information -** use the contact form at http://qt.digia.com/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 2.1 requirements -** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** In addition, as a special exception, Digia gives you certain additional -** rights. These rights are described in the Digia Qt LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef TOKENIZER_H -#define TOKENIZER_H - -#include -#include -#include - -QT_BEGIN_NAMESPACE - -template -struct QTokenizerPrivate { - using char_type = typename T::value_type; - - struct State { - bool inQuote = false; - bool inEscape = false; - char_type quoteChar = '\0'; - }; - - QTokenizerPrivate(const T& _string, const T& _delims) : - string(_string) - , begin(string.begin()) - , end(string.end()) - , tokenBegin(end) - , tokenEnd(begin) - , delimiters(_delims) - { - } - - [[nodiscard]] bool isDelimiter(char_type c) const { - return delimiters.contains(c); - } - - [[nodiscard]] bool isQuote(char_type c) const { - return quotes.contains(c); - } - - // Returns true if a delimiter was not hit - bool nextChar(State* state, char_type c) { - if (state->inQuote) { - if (state->inEscape) { - state->inEscape = false; - } else if (c == '\\') { - state->inEscape = true; - } else if (c == state->quoteChar) { - state->inQuote = false; - } - } else { - if (isDelimiter(c)) - return false; - state->inQuote = isQuote(state->quoteChar = c); - } - return true; - } - - T string; - // ### copies begin and end for performance, premature optimization? - const_iterator begin; - const_iterator end; - const_iterator tokenBegin; - const_iterator tokenEnd; - T delimiters; - T quotes; - bool isDelim = false; - bool returnDelimiters = false; - bool returnQuotes = false; -}; - -template -class QTokenizer { -public: - using char_type = typename T::value_type; - - /*! - \class QTokenizer - \inmodule QtNetwork - \brief QTokenizer tokenizes Strings on QString, QByteArray, - std::string or std::wstring - - Example Usage: - - \code - QString str = ...; - QByteArrayTokenizer tokenizer(str, "; "); - tokenizer.setQuoteCharacters("\"'"); - tokenizer.setReturnDelimiters(true); - while (tokenizer.hasNext()) { - QByteArray token = tokenizer.next(); - bool isDelimiter = tokenizer.isDelimiter(); - ... - } - \endcode - - \param string The string to tokenize - \param delimiters A string containing delimiters - - \sa QStringTokenizer, QByteArrayTokenizer, StringTokenizer, WStringTokenizer - */ - QTokenizer(const T& string, const T& delimiters) - : d(new QTokenizerPrivate(string, delimiters)) - { } - - /*! - Whether or not to return delimiters as tokens - \see setQuoteCharacters - */ - void setReturnDelimiters(bool enable) { d->returnDelimiters = enable; } - - - /*! - Sets characters that are considered to start and end quotes. - - When between two characters considered a quote, delimiters will - be ignored. - - When between quotes, blackslash characters will cause the QTokenizer - to skip the next character. - - \param quotes Characters that delimit quotes. - */ - void setQuoteCharacters(const T& quotes) { d->quotes = quotes; } - - - /*! - Whether or not to return delimiters as tokens - \see setQuoteCharacters - */ - void setReturnQuoteCharacters(bool enable) { d->returnQuotes = enable; } - - - /*! - Retrieve next token. - - Returns true if there are more tokens, false otherwise. - - \sa next() - */ - bool hasNext() - { - typename QTokenizerPrivate::State state; - d->isDelim = false; - for (;;) { - d->tokenBegin = d->tokenEnd; - if (d->tokenEnd == d->end) - return false; - d->tokenEnd++; - if (d->nextChar(&state, *d->tokenBegin)) - break; - if (d->returnDelimiters) { - d->isDelim = true; - return true; - } - } - while (d->tokenEnd != d->end && d->nextChar(&state, *d->tokenEnd)) { - d->tokenEnd++; - } - return true; - } - - /*! - Resets the tokenizer to the starting position. - */ - void reset() { - d->tokenEnd = d->begin; - } - - /*! - Returns true if the current token is a delimiter, - if one more more delimiting characters have been set. - */ - [[nodiscard]] bool isDelimiter() const { return d->isDelim; } - - /*! - Returns the current token. - - Use \c hasNext() to fetch the next token. - */ - [[nodiscard]] T next() const { - int len = std::distance(d->tokenBegin, d->tokenEnd); - const_iterator tmpStart = d->tokenBegin; - if (!d->returnQuotes && len > 1 && d->isQuote(*d->tokenBegin)) { - tmpStart++; - len -= 2; - } - return T(tmpStart, len); - } - -private: - friend class QStringTokenizer; - QSharedPointer > d; -}; - -class QStringTokenizer : public QTokenizer { -public: - QStringTokenizer(const QString &string, const QString &delim) : - QTokenizer(string, delim) {} - /** - * @brief Like \see next(), but returns a lightweight string reference - * @return A reference to the token within the string - */ - QStringRef stringRef() { - // If those differences overflow an int we'd have a veeeeeery long string in memory - int begin = std::distance(d->begin, d->tokenBegin); - int end = std::distance(d->tokenBegin, d->tokenEnd); - if (!d->returnQuotes && d->isQuote(*d->tokenBegin)) { - begin++; - end -= 2; - } - return QStringRef(&d->string, begin, end); - } -}; - -using QByteArrayTokenizer = QTokenizer; -using StringTokenizer = QTokenizer; -using WStringTokenizer = QTokenizer; - -QT_END_NAMESPACE - -#endif // TOKENIZER_H - diff --git a/src/3rdparty/qtokenizer/qtokenizer.pro b/src/3rdparty/qtokenizer/qtokenizer.pro deleted file mode 100644 index 4dcd70028..000000000 --- a/src/3rdparty/qtokenizer/qtokenizer.pro +++ /dev/null @@ -1,2 +0,0 @@ -TEMPLATE = subdirs -SUBDIRS = test diff --git a/src/3rdparty/qtokenizer/test/test.pro b/src/3rdparty/qtokenizer/test/test.pro deleted file mode 100644 index 269fcf6b0..000000000 --- a/src/3rdparty/qtokenizer/test/test.pro +++ /dev/null @@ -1,8 +0,0 @@ -TEMPLATE = app -QT += testlib -CONFIG += testlib -TARGET = test -INCLUDEPATH += . .. - -# Input -SOURCES += tst_qtokenizer.cpp diff --git a/src/3rdparty/qtokenizer/test/tst_qtokenizer.cpp b/src/3rdparty/qtokenizer/test/tst_qtokenizer.cpp deleted file mode 100644 index 537439ccf..000000000 --- a/src/3rdparty/qtokenizer/test/tst_qtokenizer.cpp +++ /dev/null @@ -1,139 +0,0 @@ -#include - -#include "qtokenizer.h" - -namespace { - const QString simple = QLatin1String("A simple tokenizer test"); - const QString quoted = QLatin1String("\"Wait for me!\" he shouted"); -} - -class TestTokenizer : public QObject -{ - Q_OBJECT -private slots: - void tokenizeQStringSimple() { - QStringTokenizer tokenizer(simple, " "); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.next(), QLatin1String("A")); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.next(), QLatin1String("simple")); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.next(), QLatin1String("tokenizer")); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.next(), QLatin1String("test")); - - QCOMPARE(tokenizer.hasNext(), false); - } - - void tokenizeQStringSimpleRef() { - QStringTokenizer tokenizer(simple, " "); - - QCOMPARE(tokenizer.hasNext(), true); - QVERIFY(tokenizer.stringRef() == QLatin1String("A")); - - QCOMPARE(tokenizer.hasNext(), true); - QVERIFY(tokenizer.stringRef() == QLatin1String("simple")); - - QCOMPARE(tokenizer.hasNext(), true); - QVERIFY(tokenizer.stringRef() == QLatin1String("tokenizer")); - - QCOMPARE(tokenizer.hasNext(), true); - QVERIFY(tokenizer.stringRef() == QLatin1String("test")); - - QCOMPARE(tokenizer.hasNext(), false); - } - - void tokenizeQStringQuoted() { - const QString multiquote(QLatin1String("\"'Billy - the Kid' is dead!\"")); - QStringTokenizer tokenizer(multiquote, " -"); - tokenizer.setQuoteCharacters("\""); - tokenizer.setReturnQuoteCharacters(true); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.next(), QLatin1String("\"'Billy - the Kid' is dead!\"")); - - QCOMPARE(tokenizer.hasNext(), false); - } - - void tokenizeQStringSkipQuotes() { - const QString multiquote(QLatin1String("\"'Billy - the Kid' is dead!\"")); - QStringTokenizer tokenizer(multiquote, " "); - tokenizer.setQuoteCharacters("\""); - tokenizer.setReturnQuoteCharacters(false); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.next(), QLatin1String("'Billy - the Kid' is dead!")); - QCOMPARE(tokenizer.stringRef().toString(), QLatin1String("'Billy - the Kid' is dead!")); - - QCOMPARE(tokenizer.hasNext(), false); - } - - - void tokenizeQStringWithDelims() { - const QString delims(QLatin1String("I;Insist,On/a-Delimiter")); - QStringTokenizer tokenizer(delims, ";,/-"); - tokenizer.setReturnDelimiters(true); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.isDelimiter(), false); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.isDelimiter(), true); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.isDelimiter(), false); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.isDelimiter(), true); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.isDelimiter(), false); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.isDelimiter(), true); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.isDelimiter(), false); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.isDelimiter(), true); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.isDelimiter(), false); - - QCOMPARE(tokenizer.hasNext(), false); - } - - void resetTokenizer() { - for (int i = 0; i < 2; i++) { - QStringTokenizer tokenizer(simple, " "); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.next(), QLatin1String("A")); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.next(), QLatin1String("simple")); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.next(), QLatin1String("tokenizer")); - - QCOMPARE(tokenizer.hasNext(), true); - QCOMPARE(tokenizer.next(), QLatin1String("test")); - - QCOMPARE(tokenizer.hasNext(), false); - - tokenizer.reset(); - } - } - - // ### QByteArray, other types -}; - -QTEST_APPLESS_MAIN(TestTokenizer) - -#include "tst_qtokenizer.moc" - diff --git a/src/cmd/CMakeLists.txt b/src/cmd/CMakeLists.txt index fad2ed6ed..d4293efb5 100644 --- a/src/cmd/CMakeLists.txt +++ b/src/cmd/CMakeLists.txt @@ -14,9 +14,6 @@ target_link_libraries(cmdCore Qt::Network ) -# Need tokenizer for netrc parser -target_include_directories(cmdCore PRIVATE ${CMAKE_SOURCE_DIR}/src/3rdparty/qtokenizer) - if(UNIX AND NOT APPLE) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIE") diff --git a/src/cmd/netrcparser.cpp b/src/cmd/netrcparser.cpp index a89eb7622..266fe4f3e 100644 --- a/src/cmd/netrcparser.cpp +++ b/src/cmd/netrcparser.cpp @@ -15,8 +15,7 @@ #include #include #include - -#include +#include #include @@ -59,33 +58,32 @@ bool NetrcParser::parse() } QString content = netrc.readAll(); - QStringTokenizer tokenizer(content, " \n\t"); - tokenizer.setQuoteCharacters("\"'"); + auto tokenizer = QStringTokenizer{content, u" \n\t"}; LoginPair pair; QString machine; bool isDefault = false; - while (tokenizer.hasNext()) { - QString key = tokenizer.next(); + for(auto itToken = tokenizer.cbegin(); itToken != tokenizer.cend(); ++itToken) { + const auto key = *itToken; if (key == defaultKeyword) { tryAddEntryAndClear(machine, pair, isDefault); isDefault = true; continue; // don't read a value } - if (!tokenizer.hasNext()) { + if (itToken != tokenizer.cend()) { qDebug() << "error fetching value for" << key; return false; } - QString value = tokenizer.next(); + auto value = *(++itToken); if (key == machineKeyword) { tryAddEntryAndClear(machine, pair, isDefault); - machine = value; + machine = value.toString(); } else if (key == loginKeyword) { - pair.first = value; + pair.first = value.toString(); } else if (key == passwordKeyword) { - pair.second = value; + pair.second = value.toString(); } // ignore unsupported tokens } tryAddEntryAndClear(machine, pair, isDefault);