diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 98266443..48b6aea2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -168,26 +168,6 @@ build-darwin-qa: paths: - bridge_*.tgz -build-ie-darwin: - extends: .build-darwin-base - script: - - make build-ie - artifacts: - name: "ie-darwin-$CI_COMMIT_SHORT_SHA" - paths: - - ie_*.tgz - -build-ie-darwin-qa: - extends: .build-darwin-base - only: - - web - script: - - BUILD_TAGS="build_qa" make build-ie - artifacts: - name: "ie-darwin-qa-$CI_COMMIT_SHORT_SHA" - paths: - - ie_*.tgz - # Stage: MIRROR mirror-repo: diff --git a/Makefile b/Makefile index 516c687c..da99e87c 100644 --- a/Makefile +++ b/Makefile @@ -7,29 +7,16 @@ TARGET_CMD?=Desktop-Bridge TARGET_OS?=${GOOS} ## Build -.PHONY: build build-ie build-nogui build-ie-nogui build-launcher build-launcher-ie versioner hasher +.PHONY: build build-nogui build-launcher versioner hasher # Keep version hardcoded so app build works also without Git repository. BRIDGE_APP_VERSION?=1.8.12+git -IE_APP_VERSION?=1.3.3+git APP_VERSION:=${BRIDGE_APP_VERSION} SRC_ICO:=logo.ico SRC_ICNS:=Bridge.icns SRC_SVG:=logo.svg -TGT_ICNS:=Bridge.icns EXE_NAME:=proton-bridge CONFIGNAME:=bridge -WINDRES_DEFINE:=BUILD_BRIDGE -ifeq "${TARGET_CMD}" "Import-Export" - APP_VERSION:=${IE_APP_VERSION} - SRC_ICO:=ie.ico - SRC_ICNS:=ie.icns - SRC_SVG:=ie.svg - TGT_ICNS:=ImportExport.icns - EXE_NAME:=proton-ie - CONFIGNAME:=importExport - WINDRES_DEFINE:=BUILD_IE -endif REVISION:=$(shell git rev-parse --short=10 HEAD) BUILD_TIME:=$(shell date +%FT%T%z) @@ -41,7 +28,6 @@ ifneq "${BUILD_LDFLAGS}" "" GO_LDFLAGS+=${BUILD_LDFLAGS} endif GO_LDFLAGS_LAUNCHER:=${GO_LDFLAGS} -GO_LDFLAGS_LAUNCHER+=$(addprefix -X main.,ConfigName=${CONFIGNAME} ExeName=proton-${APP}) ifeq "${TARGET_OS}" "windows" GO_LDFLAGS_LAUNCHER+=-H=windowsgui endif @@ -70,9 +56,6 @@ EXE_TARGET:=${DEPLOY_DIR}/${TARGET_OS}/${EXE} EXE_QT_TARGET:=${DEPLOY_DIR}/${TARGET_OS}/${EXE_QT} TGZ_TARGET:=bridge_${TARGET_OS}_${REVISION}.tgz -ifeq "${TARGET_CMD}" "Import-Export" - TGZ_TARGET:=ie_${TARGET_OS}_${REVISION}.tgz -endif ifdef QT_API VENDOR_TARGET:=prepare-vendor update-qt-docs @@ -82,15 +65,9 @@ endif build: ${TGZ_TARGET} -build-ie: - TARGET_CMD=Import-Export $(MAKE) build - build-nogui: gofiles go build ${BUILD_FLAGS} -o ${EXE_NAME} cmd/${TARGET_CMD}/main.go -build-ie-nogui: - TARGET_CMD=Import-Export $(MAKE) build-nogui - ifeq "${GOOS}" "windows" PRERESOURCECMD:=cp ./resource.syso ./cmd/launcher/resource.syso POSTRESOURCECMD:=rm -f ./cmd/launcher/resource.syso @@ -100,9 +77,6 @@ build-launcher: ${RESOURCE_FILE} go build ${BUILD_FLAGS_LAUNCHER} -o launcher-${EXE} ./cmd/launcher/ ${POSTRESOURCECMD} -build-launcher-ie: - TARGET_CMD=Import-Export $(MAKE) build-launcher - versioner: go build ${BUILD_FLAGS} -o versioner utils/versioner/main.go @@ -124,7 +98,7 @@ ${DEPLOY_DIR}/darwin: ${EXE_TARGET} mv ${EXE_TARGET}/Contents/MacOS/{${DIRNAME},${EXE_NAME}}; \ perl -i -pe"s/>${DIRNAME}/>${EXE_NAME}/g" ${EXE_TARGET}/Contents/Info.plist; \ fi - cp ./internal/frontend/share/icons/${SRC_ICNS} ${DARWINAPP_CONTENTS}/Resources/${TGT_ICNS} + cp ./internal/frontend/share/icons/${SRC_ICNS} ${DARWINAPP_CONTENTS}/Resources/${SRC_ICNS} cp LICENSE ${DARWINAPP_CONTENTS}/Resources/ rm -rf "${DARWINAPP_CONTENTS}/Frameworks/QtWebEngine.framework" rm -rf "${DARWINAPP_CONTENTS}/Frameworks/QtWebView.framework" @@ -155,7 +129,7 @@ WINDRES_YEAR:=$(shell date +%Y) APP_VERSION_COMMA:=$(shell echo "${APP_VERSION}" | sed -e 's/[^0-9,.]*//g' -e 's/\./,/g') resource.syso: ./internal/frontend/share/info.rc ./internal/frontend/share/icons/${SRC_ICO} .FORCE rm -f ./*.syso - windres --target=pe-x86-64 -I ./internal/frontend/share/icons/ -D ${WINDRES_DEFINE} -D ICO_FILE=${SRC_ICO} -D EXE_NAME="${EXE_NAME}" -D FILE_VERSION="${APP_VERSION}" -D ORIGINAL_FILE_NAME="${EXE}" -D PRODUCT_VERSION="${APP_VERSION}" -D FILE_VERSION_COMMA=${APP_VERSION_COMMA} -D YEAR=${WINDRES_YEAR} -o $@ $< + windres --target=pe-x86-64 -I ./internal/frontend/share/icons/ -D ICO_FILE=${SRC_ICO} -D EXE_NAME="${EXE_NAME}" -D FILE_VERSION="${APP_VERSION}" -D ORIGINAL_FILE_NAME="${EXE}" -D PRODUCT_VERSION="${APP_VERSION}" -D FILE_VERSION_COMMA=${APP_VERSION_COMMA} -D YEAR=${WINDRES_YEAR} -o $@ $< ## Rules for therecipe/qt .PHONY: prepare-vendor update-vendor update-qt-docs @@ -232,13 +206,11 @@ test: gofiles ./internal/events/... \ ./internal/frontend/cli/... \ ./internal/imap/... \ - ./internal/importexport/... \ ./internal/locations/... \ ./internal/logging/... \ ./internal/metrics/... \ ./internal/smtp/... \ ./internal/store/... \ - ./internal/transfer/... \ ./internal/updater/... \ ./internal/users/... \ ./internal/versioner/... \ @@ -258,7 +230,6 @@ integration-test-bridge: mocks: mockgen --package mocks github.com/ProtonMail/proton-bridge/internal/users Locator,PanicHandler,CredentialsStorer,StoreMaker > internal/users/mocks/mocks.go mockgen --package mocks github.com/ProtonMail/proton-bridge/pkg/listener Listener > internal/users/mocks/listener_mocks.go - mockgen --package mocks github.com/ProtonMail/proton-bridge/internal/transfer PanicHandler,IMAPClientProvider > internal/transfer/mocks/mocks.go mockgen --package mocks github.com/ProtonMail/proton-bridge/internal/store PanicHandler,BridgeUser,ChangeNotifier > internal/store/mocks/mocks.go mockgen --package mocks github.com/ProtonMail/proton-bridge/pkg/listener Listener > internal/store/mocks/utils_mocks.go mockgen --package mocks github.com/ProtonMail/proton-bridge/pkg/pmapi Client,Manager > pkg/pmapi/mocks/mocks.go @@ -284,7 +255,7 @@ updates: install-go-mod-outdated doc: godoc -http=:6060 -release-notes: release-notes/bridge_stable.html release-notes/bridge_early.html release-notes/ie_stable.html release-notes/ie_early.html +release-notes: release-notes/bridge_stable.html release-notes/bridge_early.html release-notes/%.html: release-notes/%.md ./utils/release_notes.sh $^ @@ -292,21 +263,17 @@ release-notes/%.html: release-notes/%.md .PHONY: gofiles # Following files are for the whole app so it makes sense to have them in bridge package. # (Options like cmd or internal were considered and bridge package is the best place for them.) -gofiles: ./internal/bridge/credits.go ./internal/importexport/credits.go +gofiles: ./internal/bridge/credits.go ./internal/bridge/credits.go: ./utils/credits.sh go.mod cd ./utils/ && ./credits.sh bridge -./internal/importexport/credits.go: ./utils/credits.sh go.mod - cd ./utils/ && ./credits.sh importexport - ## Run and debug -.PHONY: run run-qt run-qt-cli run-nogui run-nogui-cli run-debug run-qml-preview run-ie-qml-preview run-ie run-ie-qt run-ie-qt-cli run-ie-nogui run-ie-nogui-cli clean-vendor clean-frontend-qt clean-frontend-qt-ie clean-frontend-qt-common clean +.PHONY: run run-qt run-qt-cli run-nogui run-nogui-cli run-debug run-qml-preview clean-vendor clean-frontend-qt clean-frontend-qt-common clean LOG?=debug LOG_IMAP?=client # client/server/all, or empty to turn it off LOG_SMTP?=--log-smtp # empty to turn it off RUN_FLAGS?=-m -l=${LOG} --log-imap=${LOG_IMAP} ${LOG_SMTP} -RUN_FLAGS_IE?=-m -l=${LOG} run: run-nogui-cli @@ -325,24 +292,13 @@ run-debug: run-qml-preview: $(MAKE) -C internal/frontend/qt -f Makefile.local qmlpreview -run-ie-qml-preview: - $(MAKE) -C internal/frontend/qt-ie -f Makefile.local qmlpreview - -run-ie: - TARGET_CMD=Import-Export RUN_FLAGS="${RUN_FLAGS_IE}" $(MAKE) run -run-ie-qt: - TARGET_CMD=Import-Export RUN_FLAGS="${RUN_FLAGS_IE}" $(MAKE) run-qt -run-ie-nogui: - TARGET_CMD=Import-Export RUN_FLAGS="${RUN_FLAGS_IE}" $(MAKE) run-nogui clean-frontend-qt: - $(MAKE) -C internal/frontend/qt -f Makefile.local clean -clean-frontend-qt-ie: - $(MAKE) -C internal/frontend/qt-ie -f Makefile.local clean + # TODO: $(MAKE) -C internal/frontend/qt -f Makefile.local clean clean-frontend-qt-common: - $(MAKE) -C internal/frontend/qt-common -f Makefile.local clean + # TODO: $(MAKE) -C internal/frontend/qt-common -f Makefile.local clean -clean-vendor: clean-frontend-qt clean-frontend-qt-ie clean-frontend-qt-common +clean-vendor: clean-frontend-qt clean-frontend-qt-common rm -rf ./vendor clean: clean-vendor diff --git a/README.md b/README.md index 56645a3d..442b06ab 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ check the results. More details [on the public website](https://protonmail.com/import-export). +The Import-Export app is developed in separate branch `master-ie`. + ## Launchers Launchers are binaries used to run the ProtonMail Bridge or Import-Export apps. @@ -69,7 +71,6 @@ or ### Integration testing - `TEST_ENV`: set which env to use (fake or live) -- `TEST_APP`: set which app to test (bridge or ie) - `TEST_ACCOUNTS`: set JSON file with configured accounts - `TAGS`: set build tags for tests - `FEATURES`: set feature dir, file or scenario to test diff --git a/cmd/launcher/main.go b/cmd/launcher/main.go index 49c38877..a5d2304f 100644 --- a/cmd/launcher/main.go +++ b/cmd/launcher/main.go @@ -38,11 +38,10 @@ import ( "github.com/sirupsen/logrus" ) -const appName = "ProtonMail Launcher" - -var ( - ConfigName = "" // nolint[gochecknoglobals] - ExeName = "" // nolint[gochecknoglobals] +const ( + appName = "ProtonMail Launcher" + configName = "bridge" + exeName = "proton-bridge" ) func main() { // nolint[funlen] @@ -51,12 +50,12 @@ func main() { // nolint[funlen] crashHandler := crash.NewHandler(reporter.ReportException) defer crashHandler.HandlePanic() - locationsProvider, err := locations.NewDefaultProvider(filepath.Join(constants.VendorName, ConfigName)) + locationsProvider, err := locations.NewDefaultProvider(filepath.Join(constants.VendorName, configName)) if err != nil { logrus.WithError(err).Fatal("Failed to get locations provider") } - locations := locations.New(locationsProvider, ConfigName) + locations := locations.New(locationsProvider, configName) logsPath, err := locations.ProvideLogsPath() if err != nil { @@ -87,9 +86,9 @@ func main() { // nolint[funlen] versioner := versioner.New(updatesPath) - exe, err := getPathToUpdatedExecutable(ExeName, versioner, kr, reporter) + exe, err := getPathToUpdatedExecutable(exeName, versioner, kr, reporter) if err != nil { - if exe, err = getFallbackExecutable(ExeName, versioner); err != nil { + if exe, err = getFallbackExecutable(exeName, versioner); err != nil { logrus.WithError(err).Fatal("Failed to find any launchable executable") } } diff --git a/dist/proton-ie.desktop b/dist/proton-ie.desktop deleted file mode 100644 index 856174c9..00000000 --- a/dist/proton-ie.desktop +++ /dev/null @@ -1,11 +0,0 @@ -[Desktop Entry] -Type=Application -Version=1.1 -Name=ProtonMail Import-Export app -GenericName=ProtonMail Import-Export app for Linux -Comment=The Import-Export app helps you to migrate your emails from local files or remote IMAP servers to ProtonMail or simply export emails to local folder. -Icon=protonmail-import-export-app -Exec=protonmail-import-export-app -Terminal=false -Categories=Office;Email;Network -StartupWMClass=protonmail-import-export-app diff --git a/doc/importexport.md b/doc/importexport.md deleted file mode 100644 index c50a989e..00000000 --- a/doc/importexport.md +++ /dev/null @@ -1,135 +0,0 @@ -# Import-Export app - -## Main blocks - -This is basic overview of the main Import-Export blocks. - -```mermaid -graph LR - S[ProtonMail server] - U[User] - - subgraph "Import-Export app" - Users - Frontend["Qt / CLI"] - ImportExport - Transfer - - Frontend --> ImportExport - Frontend --> Transfer - ImportExport --> Users - ImportExport --> Transfer - end - - EML --> Transfer - MBOX --> Transfer - IMAP --> Transfer - S --> Transfer - - Transfer --> EML - Transfer --> MBOX - Transfer --> S - - U --> Frontend -``` - -## Code structure - -More detailed graph of main types used in Import-Export app and connection between them. - -```mermaid -graph TD - PM[ProtonMail Server] - EML[EML] - MBOX[MBOX] - IMAP[IMAP] - - subgraph "Import-Export app" - subgraph "pkg users" - subgraph "pkg credentials" - CredStore[Store] - Creds[Credentials] - - CredStore --> Creds - end - - US[Users] - U[User] - - US --> U - end - - subgraph "pkg frontend" - CLI - Qt - end - - subgraph "pkg importExport" - IE[ImportExport] - end - - subgraph "pkg transfer" - Transfer - Rules - Progress - - Provider - LocalProvider - EMLProvider - MBOXProvider - IMAPProvider - PMAPIProvider - - Mailbox - Message - - Transfer --> |source|Provider - Transfer --> |target|Provider - Transfer --> Rules - Transfer --> Progress - - Provider --> LocalProvider - Provider --> EMLProvider - Provider --> MBOXProvider - Provider --> IMAPProvider - Provider --> PMAPIProvider - - LocalProvider --> EMLProvider - LocalProvider --> MBOXProvider - - Provider --> Mailbox - Provider --> Message - - end - - subgraph PMAPI - APIM[ClientManager] - APIC[Client] - - APIM --> APIC - end - end - - CLI --> IE - CLI --> Transfer - CLI --> Progress - Qt --> IE - Qt --> Transfer - Qt --> Progress - - U --> CredStore - U --> Creds - - US --> APIM - U --> APIM - - PMAPIProvider --> APIM - EMLProvider --> EML - MBOXProvider --> MBOX - IMAPProvider --> IMAP - - IE --> US - IE --> Transfer - - APIC --> PM -``` diff --git a/doc/index.md b/doc/index.md index 66da2f5f..d46f2cb4 100644 --- a/doc/index.md +++ b/doc/index.md @@ -1,14 +1,9 @@ -# Documentation +# Bridge Documentation Documentation pages in order to read for a novice: -## Bridge - * [Bridge code](bridge.md) * [Internal Bridge database](database.md) * [Communication between Bridge, Client and Server](communication.md) * [Encryption](encryption.md) -## Import-Export app - -* [Import-Export code](importexport.md) diff --git a/internal/bridge/bridge.go b/internal/bridge/bridge.go index b8c7bf29..4a66774a 100644 --- a/internal/bridge/bridge.go +++ b/internal/bridge/bridge.go @@ -68,8 +68,15 @@ func New( clientManager.AllowProxy() } - storeFactory := newStoreFactory(cache, sentryReporter, panicHandler, eventListener) - u := users.New(locations, panicHandler, eventListener, clientManager, credStorer, storeFactory, true) + u := users.New( + locations, + panicHandler, + eventListener, + clientManager, + credStorer, + newStoreFactory(cache, sentryReporter, panicHandler, eventListener), + ) + b := &Bridge{ Users: u, diff --git a/internal/frontend/share/info.rc b/internal/frontend/share/info.rc new file mode 100644 index 00000000..69438d91 --- /dev/null +++ b/internal/frontend/share/info.rc @@ -0,0 +1,36 @@ +#define STRINGIZE_(x) #x +#define STRINGIZE(x) STRINGIZE_(x) + +IDI_ICON1 ICON DISCARDABLE STRINGIZE(ICO_FILE) + +#define FILE_COMMENTS "The Bridge is an application that runs on your computer in the background and seamlessly encrypts and decrypts your mail as it enters and leaves your computer." +#define FILE_DESCRIPTION "ProtonMail Bridge" +#define INTERNAL_NAME STRINGIZE(EXE_NAME) +#define PRODUCT_NAME "ProtonMail Bridge for Windows" + +#define LEGAL_COPYRIGHT "(C) " STRINGIZE(YEAR) " Proton Technologies AG" + +1 VERSIONINFO +FILEVERSION FILE_VERSION_COMMA,0 +PRODUCTVERSION FILE_VERSION_COMMA,0 +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "Comments", FILE_COMMENTS + VALUE "CompanyName", "Proton Technologies AG" + VALUE "FileDescription", FILE_DESCRIPTION + VALUE "FileVersion", STRINGIZE(FILE_VERSION) + VALUE "InternalName", INTERNAL_NAME + VALUE "LegalCopyright", LEGAL_COPYRIGHT + VALUE "OriginalFilename", STRINGIZE(ORIGINAL_FILE_NAME) + VALUE "ProductName", PRODUCT_NAME + VALUE "ProductVersion", STRINGIZE(PRODUCT_VERSION) + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0409, 0x04B0 + END +END diff --git a/internal/importexport/importexport.go b/internal/importexport/importexport.go deleted file mode 100644 index c7e38141..00000000 --- a/internal/importexport/importexport.go +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -// Package importexport provides core functionality of Import-Export app. -package importexport - -import ( - "bytes" - "context" - - "github.com/ProtonMail/proton-bridge/internal/events" - "github.com/ProtonMail/proton-bridge/internal/transfer" - "github.com/ProtonMail/proton-bridge/internal/users" - "github.com/ProtonMail/proton-bridge/pkg/pmapi" - - "github.com/ProtonMail/proton-bridge/pkg/listener" - logrus "github.com/sirupsen/logrus" -) - -var ( - log = logrus.WithField("pkg", "importexport") //nolint[gochecknoglobals] -) - -type ImportExport struct { - *users.Users - - locations Locator - cache Cacher - panicHandler users.PanicHandler - eventListener listener.Listener - clientManager pmapi.Manager -} - -func New( - locations Locator, - cache Cacher, - panicHandler users.PanicHandler, - eventListener listener.Listener, - clientManager pmapi.Manager, - credStorer users.CredentialsStorer, -) *ImportExport { - u := users.New(locations, panicHandler, eventListener, clientManager, credStorer, &storeFactory{}, false) - - return &ImportExport{ - Users: u, - - locations: locations, - cache: cache, - panicHandler: panicHandler, - eventListener: eventListener, - clientManager: clientManager, - } -} - -// ReportBug reports a new bug from the user. -func (ie *ImportExport) ReportBug(osType, osVersion, description, accountName, address, emailClient string) error { - return ie.clientManager.ReportBug(context.Background(), pmapi.ReportBugReq{ - OS: osType, - OSVersion: osVersion, - Browser: emailClient, - Title: "[Import-Export] Bug", - Description: description, - Username: accountName, - Email: address, - }) -} - -// ReportFile submits import report file. -func (ie *ImportExport) ReportFile(osType, osVersion, accountName, address string, logdata []byte) error { - report := pmapi.ReportBugReq{ - OS: osType, - OSVersion: osVersion, - Description: "An Import-Export report from the user swam down the river.", - Title: "[Import-Export] report file", - Username: accountName, - Email: address, - } - - report.AddAttachment("log", "report.log", bytes.NewReader(logdata)) - - return ie.clientManager.ReportBug(context.Background(), report) -} - -// GetLocalImporter returns transferrer from local EML or MBOX structure to ProtonMail account. -func (ie *ImportExport) GetLocalImporter(username, address, path string) (*transfer.Transfer, error) { - source := transfer.NewLocalProvider(path) - target, err := ie.getPMAPIProvider(username, address) - if err != nil { - return nil, err - } - logsPath, err := ie.locations.ProvideLogsPath() - if err != nil { - return nil, err - } - return transfer.New(ie.panicHandler, newImportMetricsManager(ie), logsPath, ie.cache.GetTransferDir(), source, target) -} - -// GetRemoteImporter returns transferrer from remote IMAP to ProtonMail account. -func (ie *ImportExport) GetRemoteImporter(username, address, remoteUsername, remotePassword, host, port string) (*transfer.Transfer, error) { - source, err := transfer.NewIMAPProvider(remoteUsername, remotePassword, host, port) - if err != nil { - return nil, err - } - target, err := ie.getPMAPIProvider(username, address) - if err != nil { - return nil, err - } - logsPath, err := ie.locations.ProvideLogsPath() - if err != nil { - return nil, err - } - return transfer.New(ie.panicHandler, newImportMetricsManager(ie), logsPath, ie.cache.GetTransferDir(), source, target) -} - -// GetEMLExporter returns transferrer from ProtonMail account to local EML structure. -func (ie *ImportExport) GetEMLExporter(username, address, path string) (*transfer.Transfer, error) { - source, err := ie.getPMAPIProvider(username, address) - if err != nil { - return nil, err - } - target := transfer.NewEMLProvider(path) - logsPath, err := ie.locations.ProvideLogsPath() - if err != nil { - return nil, err - } - return transfer.New(ie.panicHandler, newExportMetricsManager(ie), logsPath, ie.cache.GetTransferDir(), source, target) -} - -// GetMBOXExporter returns transferrer from ProtonMail account to local MBOX structure. -func (ie *ImportExport) GetMBOXExporter(username, address, path string) (*transfer.Transfer, error) { - source, err := ie.getPMAPIProvider(username, address) - if err != nil { - return nil, err - } - target := transfer.NewMBOXProvider(path) - logsPath, err := ie.locations.ProvideLogsPath() - if err != nil { - return nil, err - } - return transfer.New(ie.panicHandler, newExportMetricsManager(ie), logsPath, ie.cache.GetTransferDir(), source, target) -} - -func (ie *ImportExport) getPMAPIProvider(username, address string) (*transfer.PMAPIProvider, error) { - user, err := ie.Users.GetUser(username) - if err != nil { - return nil, err - } - - addressID, err := user.GetAddressID(address) - if err != nil { - log.WithError(err).Info("Address does not exist, using all addresses") - } - - provider, err := transfer.NewPMAPIProvider(user.GetClient(), user.ID(), addressID) - if err != nil { - return nil, err - } - - go func() { - internetOffCh := ie.eventListener.ProvideChannel(events.InternetOffEvent) - internetOnCh := ie.eventListener.ProvideChannel(events.InternetOnEvent) - for { - select { - case <-internetOffCh: - provider.SetConnectionDown() - case <-internetOnCh: - provider.SetConnectionUp() - } - } - }() - - return provider, nil -} diff --git a/internal/importexport/metrics.go b/internal/importexport/metrics.go deleted file mode 100644 index 9bab845f..00000000 --- a/internal/importexport/metrics.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package importexport - -import ( - "strconv" - - "github.com/ProtonMail/proton-bridge/internal/metrics" - "github.com/sirupsen/logrus" -) - -type metricsManager struct { - ie *ImportExport - category metrics.Category -} - -func newImportMetricsManager(ie *ImportExport) *metricsManager { - return &metricsManager{ - ie: ie, - category: metrics.Import, - } -} - -func newExportMetricsManager(ie *ImportExport) *metricsManager { - return &metricsManager{ - ie: ie, - category: metrics.Export, - } -} - -func (m *metricsManager) Load(numberOfMailboxes int) { - label := strconv.Itoa(numberOfMailboxes) - if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferLoad, metrics.Label(label))); err != nil { - logrus.WithError(err).Error("Failed to send metric") - } -} - -func (m *metricsManager) Start() { - if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferStart, metrics.NoLabel)); err != nil { - logrus.WithError(err).Error("Failed to send metric") - } -} - -func (m *metricsManager) Complete() { - if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferComplete, metrics.NoLabel)); err != nil { - logrus.WithError(err).Error("Failed to send metric") - } -} - -func (m *metricsManager) Cancel() { - if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferCancel, metrics.NoLabel)); err != nil { - logrus.WithError(err).Error("Failed to send metric") - } -} - -func (m *metricsManager) Fail() { - if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferFail, metrics.NoLabel)); err != nil { - logrus.WithError(err).Error("Failed to send metric") - } -} diff --git a/internal/importexport/store_factory.go b/internal/importexport/store_factory.go deleted file mode 100644 index f60ef355..00000000 --- a/internal/importexport/store_factory.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package importexport - -import ( - "github.com/ProtonMail/proton-bridge/internal/store" -) - -// storeFactory implements dummy factory creating no store (not needed by Import-Export). -type storeFactory struct{} - -// New does nothing. -func (f *storeFactory) New(user store.BridgeUser) (*store.Store, error) { - return nil, nil -} - -// Remove does nothing. -func (f *storeFactory) Remove(userID string) error { - return nil -} diff --git a/internal/importexport/types.go b/internal/importexport/types.go deleted file mode 100644 index 763a23ca..00000000 --- a/internal/importexport/types.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package importexport - -type Locator interface { - ProvideLogsPath() (string, error) - Clear() error -} - -type Cacher interface { - GetTransferDir() string -} diff --git a/internal/transfer/mailbox.go b/internal/transfer/mailbox.go deleted file mode 100644 index b96c7ad0..00000000 --- a/internal/transfer/mailbox.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "crypto/sha256" - "fmt" - "strings" - - "github.com/ProtonMail/proton-bridge/pkg/pmapi" -) - -var systemFolderMapping = map[string]string{ //nolint[gochecknoglobals] - "bin": "Trash", - "junk": "Spam", - "all": "All Mail", - "sent mail": "Sent", - "draft": "Drafts", - "important": "Starred", - // Add more translations. -} - -// LeastUsedColor is intended to return color for creating a new inbox or label. -func LeastUsedColor(mailboxes []Mailbox) string { - usedColors := []string{} - for _, m := range mailboxes { - usedColors = append(usedColors, m.Color) - } - return pmapi.LeastUsedColor(usedColors) -} - -// Mailbox is universal data holder of mailbox details for every provider. -type Mailbox struct { - ID string - Name string - Color string - IsExclusive bool -} - -// IsSystemFolder returns true when ID corresponds to PM system folder. -func (m Mailbox) IsSystemFolder() bool { - return pmapi.IsSystemLabel(m.ID) -} - -// Hash returns unique identifier to be used for matching. -func (m Mailbox) Hash() string { - return fmt.Sprintf("%x", sha256.Sum256([]byte(m.Name))) -} - -// findMatchingMailboxes returns all matching mailboxes from `mailboxes`. -// Only one exclusive mailbox is included. -func (m Mailbox) findMatchingMailboxes(mailboxes []Mailbox) []Mailbox { - nameVariants := m.nameVariants() - isExclusiveIncluded := false - matches := []Mailbox{} - for i := range nameVariants { - nameVariant := nameVariants[len(nameVariants)-1-i] - for _, mailbox := range mailboxes { - if mailbox.IsExclusive && isExclusiveIncluded { - continue - } - if strings.ToLower(mailbox.Name) == nameVariant { - matches = append(matches, mailbox) - if mailbox.IsExclusive { - isExclusiveIncluded = true - } - } - } - } - return matches -} - -// nameVariants returns all possible variants of the mailbox name. -// The best match (original name) is at the end of the slice. -// Variants are all in lower case. Examples: -// * Foo/bar -> [foo, bar, foo/bar] -// * x/Bin -> [x, trash, bin, x/bin] -// * a|b/c -> [a, b, c, a|b/c] -func (m Mailbox) nameVariants() (nameVariants []string) { - name := strings.ToLower(m.Name) - if strings.Contains(name, "/") || strings.Contains(name, "|") { - for _, slashPart := range strings.Split(name, "/") { - for _, part := range strings.Split(slashPart, "|") { - if mappedPart, ok := systemFolderMapping[part]; ok { - nameVariants = append(nameVariants, strings.ToLower(mappedPart)) - } - nameVariants = append(nameVariants, part) - } - } - } - if mappedName, ok := systemFolderMapping[name]; ok { - nameVariants = append(nameVariants, strings.ToLower(mappedName)) - } - return append(nameVariants, name) -} diff --git a/internal/transfer/mailbox_test.go b/internal/transfer/mailbox_test.go deleted file mode 100644 index 068b1087..00000000 --- a/internal/transfer/mailbox_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "testing" - - r "github.com/stretchr/testify/require" -) - -func TestLeastUsedColor(t *testing.T) { - var mailboxes []Mailbox - // Unset mailboxes, should use first available color - mailboxes = nil - r.Equal(t, "#7272a7", LeastUsedColor(mailboxes)) - - // No mailboxes at all, should use first available color - mailboxes = []Mailbox{} - r.Equal(t, "#7272a7", LeastUsedColor(mailboxes)) - - // All colors have same frequency, should use first available color - mailboxes = []Mailbox{ - {Name: "Mbox1", Color: "#7272a7"}, - {Name: "Mbox2", Color: "#cf5858"}, - {Name: "Mbox3", Color: "#c26cc7"}, - {Name: "Mbox4", Color: "#7569d1"}, - {Name: "Mbox5", Color: "#69a9d1"}, - {Name: "Mbox6", Color: "#5ec7b7"}, - {Name: "Mbox7", Color: "#72bb75"}, - {Name: "Mbox8", Color: "#c3d261"}, - {Name: "Mbox9", Color: "#e6c04c"}, - {Name: "Mbox10", Color: "#e6984c"}, - {Name: "Mbox11", Color: "#8989ac"}, - {Name: "Mbox12", Color: "#cf7e7e"}, - {Name: "Mbox13", Color: "#c793ca"}, - {Name: "Mbox14", Color: "#9b94d1"}, - {Name: "Mbox15", Color: "#a8c4d5"}, - {Name: "Mbox16", Color: "#97c9c1"}, - {Name: "Mbox17", Color: "#9db99f"}, - {Name: "Mbox18", Color: "#c6cd97"}, - {Name: "Mbox19", Color: "#e7d292"}, - {Name: "Mbox20", Color: "#dfb286"}, - } - r.Equal(t, "#7272a7", LeastUsedColor(mailboxes)) - - // First three colors already used, but others wasn't. Should use first non-used one. - mailboxes = []Mailbox{ - {Name: "Mbox1", Color: "#7272a7"}, - {Name: "Mbox2", Color: "#cf5858"}, - {Name: "Mbox3", Color: "#c26cc7"}, - } - r.Equal(t, "#7569d1", LeastUsedColor(mailboxes)) -} - -func TestFindMatchingMailboxes(t *testing.T) { - mailboxes := []Mailbox{ - {Name: "Inbox", IsExclusive: true}, - {Name: "Sent", IsExclusive: true}, - {Name: "Archive", IsExclusive: true}, - {Name: "Foo", IsExclusive: false}, - {Name: "hello/world", IsExclusive: true}, - {Name: "Hello", IsExclusive: false}, - {Name: "WORLD", IsExclusive: true}, - {Name: "Trash", IsExclusive: true}, - {Name: "Drafts", IsExclusive: true}, - } - - tests := []struct { - name string - wantNames []string - }{ - {"inbox", []string{"Inbox"}}, - {"foo", []string{"Foo"}}, - {"hello", []string{"Hello"}}, - {"world", []string{"WORLD"}}, - {"hello/world", []string{"hello/world", "Hello"}}, - {"hello|world", []string{"WORLD", "Hello"}}, - {"nomailbox", []string{}}, - {"bin", []string{"Trash"}}, - {"root/bin", []string{"Trash"}}, - {"draft", []string{"Drafts"}}, - {"root/draft", []string{"Drafts"}}, - } - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - mailbox := Mailbox{Name: tc.name} - got := mailbox.findMatchingMailboxes(mailboxes) - gotNames := []string{} - for _, m := range got { - gotNames = append(gotNames, m.Name) - } - r.Equal(t, tc.wantNames, gotNames) - }) - } -} diff --git a/internal/transfer/message.go b/internal/transfer/message.go deleted file mode 100644 index b3b6dd69..00000000 --- a/internal/transfer/message.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "fmt" - "mime" - "net/mail" - "time" -) - -// Message is data holder passed between import and export. -type Message struct { - ID string - Unread bool - Body []byte - Sources []Mailbox - Targets []Mailbox -} - -// sourceNames returns array of source mailbox names. -func (msg Message) sourceNames() (names []string) { - for _, mailbox := range msg.Sources { - names = append(names, mailbox.Name) - } - return -} - -// targetNames returns array of target mailbox names. -func (msg Message) targetNames() (names []string) { - for _, mailbox := range msg.Targets { - names = append(names, mailbox.Name) - } - return -} - -// MessageStatus holds status for message used by progress manager. -type MessageStatus struct { - eventTime time.Time // Time of adding message to the process. - sourceNames []string // Source mailbox names message is in. - SourceID string // Message ID at the source. - targetNames []string // Target mailbox names message is in. - targetID string // Message ID at the target (if any). - bodyHash string // Hash of the message body. - - skipped bool - exported bool - imported bool - exportErr error - importErr error - - // Info about message displayed to user. - // This is needed only for failed messages, but we cannot know in advance - // which message will fail. We could clear it once the message passed - // without any error. However, if we say one message takes about 100 bytes - // in average, it's about 100 MB per million of messages, which is fine. - Subject string - From string - Time time.Time -} - -func (status *MessageStatus) String() string { - return fmt.Sprintf("%s (%s, %s, %s): %s", status.SourceID, status.Subject, status.From, status.Time, status.GetErrorMessage()) -} - -func (status *MessageStatus) setDetailsFromHeader(header mail.Header) { - dec := &mime.WordDecoder{} - - status.Subject = header.Get("subject") - if subject, err := dec.Decode(status.Subject); err == nil { - status.Subject = subject - } - - status.From = header.Get("from") - if from, err := dec.Decode(status.From); err == nil { - status.From = from - } - - if msgTime, err := header.Date(); err == nil { - status.Time = msgTime - } -} - -func (status *MessageStatus) hasError(includeMissing bool) bool { - return status.exportErr != nil || status.importErr != nil || (includeMissing && !status.skipped && !status.imported) -} - -// GetErrorMessage returns error message. -func (status *MessageStatus) GetErrorMessage() string { - return status.getErrorMessage(true) -} - -func (status *MessageStatus) getErrorMessage(includeMissing bool) string { - if status.skipped { - return "" - } - if status.exportErr != nil { - return fmt.Sprintf("failed to export: %s", status.exportErr) - } - if status.importErr != nil { - return fmt.Sprintf("failed to import: %s", status.importErr) - } - if includeMissing && !status.imported { - if !status.exported { - return "failed to import: lost before read" - } - return "failed to import: lost in the process" - } - return "" -} diff --git a/internal/transfer/progress.go b/internal/transfer/progress.go deleted file mode 100644 index c68c361c..00000000 --- a/internal/transfer/progress.go +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "crypto/sha256" - "fmt" - "sync" - "time" - - "github.com/sirupsen/logrus" -) - -// Progress maintains progress between import, export and user interface. -// Import and export update progress about processing messages and progress -// informs user interface, vice versa action (such as pause or resume) from -// user interface is passed down to import and export. -type Progress struct { //nolint[maligned] - log *logrus.Entry - lock sync.Locker - - updateCh chan struct{} - messageCounted bool - messageCounts map[string]uint - messageStatuses map[string]*MessageStatus - pauseReason string - isStopped bool - fatalError error - fileReport *fileReport -} - -func newProgress(log *logrus.Entry, fileReport *fileReport) Progress { - return Progress{ - log: log, - lock: &sync.Mutex{}, - - updateCh: make(chan struct{}), - messageCounts: map[string]uint{}, - messageStatuses: map[string]*MessageStatus{}, - fileReport: fileReport, - } -} - -// update is helper to notify listener for updates. -func (p *Progress) update() { - if p.updateCh == nil { - return - } - - // In case no one listens for an update, do not block the whole progress. - go func() { - defer func() { - // updateCh can be closed at the end of progress which is fine. - if r := recover(); r != nil { - log.WithField("r", r).Warn("Failed to send update") - } - }() - - select { - case p.updateCh <- struct{}{}: - case <-time.After(5 * time.Millisecond): - } - }() -} - -// finish should be called as the last call once everything is done. -func (p *Progress) finish() { - p.lock.Lock() - defer p.lock.Unlock() - - log.Debug("Progress finished") - p.cleanUpdateCh() -} - -// fatal should be called once there is error with no possible continuation. -func (p *Progress) fatal(err error) { - p.lock.Lock() - defer p.lock.Unlock() - - log.WithError(err).Error("Progress finished") - p.setStop() - p.fatalError = err - p.cleanUpdateCh() -} - -func (p *Progress) cleanUpdateCh() { - if p.updateCh == nil { - return - } - - close(p.updateCh) - p.updateCh = nil -} - -func (p *Progress) countsFinal() { - p.lock.Lock() - defer p.lock.Unlock() - defer p.update() - - log.Info("Estimating count finished") - p.messageCounted = true -} - -func (p *Progress) updateCount(mailbox string, count uint) { - p.lock.Lock() - defer p.lock.Unlock() - defer p.update() - - log.WithField("mailbox", mailbox).WithField("count", count).Debug("Mailbox count updated") - p.messageCounts[mailbox] = count -} - -// addMessage should be called as soon as there is ID of the message. -func (p *Progress) addMessage(messageID string, sourceNames, targetNames []string) { - p.lock.Lock() - defer p.lock.Unlock() - defer p.update() - - p.log.WithField("id", messageID).Trace("Message added") - p.messageStatuses[messageID] = &MessageStatus{ - eventTime: time.Now(), - sourceNames: sourceNames, - SourceID: messageID, - targetNames: targetNames, - } -} - -// messageSkipped should be called once the message is skipped due to some -// filter such as time or folder and so on. -func (p *Progress) messageSkipped(messageID string) { - p.lock.Lock() - defer p.lock.Unlock() - defer p.update() - - p.log.WithField("id", messageID).Debug("Message skipped") - - p.messageStatuses[messageID].skipped = true - p.logMessage(messageID) -} - -// messageExported should be called right before message is exported. -func (p *Progress) messageExported(messageID string, body []byte, err error) { - p.lock.Lock() - defer p.lock.Unlock() - defer p.update() - - log := p.log.WithField("id", messageID) - if err != nil { - log = log.WithError(err) - } - log.Debug("Message exported") - - status := p.messageStatuses[messageID] - status.exportErr = err - if err == nil { - status.exported = true - } - - if len(body) > 0 { - status.bodyHash = fmt.Sprintf("%x", sha256.Sum256(body)) - - if header, err := getMessageHeader(body); err != nil { - log.WithError(err).Warning("Failed to parse headers for reporting") - } else { - status.setDetailsFromHeader(header) - } - } - - // If export failed, no other step will be done with message and we can log it to the report file. - if err != nil { - p.logMessage(messageID) - } -} - -// messageImported should be called right after message is imported. -func (p *Progress) messageImported(messageID, importID string, err error) { - p.lock.Lock() - defer p.lock.Unlock() - defer p.update() - - log := p.log.WithField("id", messageID) - if err != nil { - log = log.WithError(err) - } - log.Debug("Message imported") - - p.messageStatuses[messageID].targetID = importID - p.messageStatuses[messageID].importErr = err - if err == nil { - p.messageStatuses[messageID].imported = true - } - - // Import is the last step, now we can log the result to the report file. - p.logMessage(messageID) -} - -// logMessage writes message status to log file. -func (p *Progress) logMessage(messageID string) { - if p.fileReport == nil { - return - } - p.fileReport.writeMessageStatus(p.messageStatuses[messageID]) -} - -// callWrap calls the callback and in case of problem it pause the process. -// Then it waits for user action to fix it and click on continue or abort. -// Every function doing I/O should be wrapped by this function to provide -// stopping and pausing functionality. -func (p *Progress) callWrap(callback func() error) { - for { - if p.shouldStop() { - break - } - - err := callback() - if err == nil { - break - } - - p.Pause("paused due to " + err.Error()) - } -} - -// shouldStop is utility for providers to automatically wait during pause -// and returned value determines whether the process shouls be fully stopped. -func (p *Progress) shouldStop() bool { - for p.IsPaused() { - time.Sleep(time.Second) - } - return p.IsStopped() -} - -// GetUpdateChannel returns channel notifying any update from import or export. -func (p *Progress) GetUpdateChannel() chan struct{} { - p.lock.Lock() - defer p.lock.Unlock() - - return p.updateCh -} - -// Pause pauses the progress. -func (p *Progress) Pause(reason string) { - p.lock.Lock() - defer p.lock.Unlock() - defer p.update() - - p.log.Info("Progress paused") - p.pauseReason = reason -} - -// Resume resumes the progress. -func (p *Progress) Resume() { - p.lock.Lock() - defer p.lock.Unlock() - defer p.update() - - p.log.Info("Progress resumed") - p.pauseReason = "" -} - -// IsPaused returns whether progress is paused. -func (p *Progress) IsPaused() bool { - p.lock.Lock() - defer p.lock.Unlock() - - return p.pauseReason != "" -} - -// PauseReason returns pause reason. -func (p *Progress) PauseReason() string { - p.lock.Lock() - defer p.lock.Unlock() - - return p.pauseReason -} - -// Stop stops the process. -func (p *Progress) Stop() { - p.lock.Lock() - defer p.lock.Unlock() - defer p.update() - - p.log.Info("Progress stopped") - p.setStop() - - // Once progress is stopped, some calls might be in progress. Results from - // those calls are irrelevant so we can close update channel sooner to not - // propagate any progress to user interface anymore. - p.cleanUpdateCh() -} - -func (p *Progress) setStop() { - p.isStopped = true - p.pauseReason = "" // Clear pause to run paused code and stop it. -} - -// IsStopped returns whether progress is stopped. -func (p *Progress) IsStopped() bool { - p.lock.Lock() - defer p.lock.Unlock() - - return p.isStopped -} - -// GetFatalError returns fatal error (progress failed and did not finish). -func (p *Progress) GetFatalError() error { - p.lock.Lock() - defer p.lock.Unlock() - - return p.fatalError -} - -// GetFailedMessages returns statuses of failed messages. -func (p *Progress) GetFailedMessages() []*MessageStatus { - p.lock.Lock() - defer p.lock.Unlock() - - // Include lost messages in the process only when transfer is done. - includeMissing := p.updateCh == nil - - statuses := []*MessageStatus{} - for _, status := range p.messageStatuses { - if status.hasError(includeMissing) { - statuses = append(statuses, status) - } - } - return statuses -} - -// GetCounts returns counts of exported and imported messages. -func (p *Progress) GetCounts() ProgressCounts { - p.lock.Lock() - defer p.lock.Unlock() - - counts := ProgressCounts{} - - // Return counts only once total is estimated or the process already - // ended (for a case when it ended quickly to report it correctly). - if p.updateCh != nil && !p.messageCounted { - return counts - } - - // Include lost messages in the process only when transfer is done. - includeMissing := p.updateCh == nil - - for _, mailboxCount := range p.messageCounts { - counts.Total += mailboxCount - } - for _, status := range p.messageStatuses { - counts.Added++ - if status.skipped { - counts.Skipped++ - } - if status.exported { - counts.Exported++ - } - if status.imported { - counts.Imported++ - } - if status.hasError(includeMissing) { - counts.Failed++ - } - } - return counts -} - -// GenerateBugReport generates similar file to import log except private information. -func (p *Progress) GenerateBugReport() []byte { - bugReport := bugReport{} - for _, status := range p.messageStatuses { - bugReport.writeMessageStatus(status) - } - return bugReport.getData() -} - -// FileReport returns path to generated defailed file report. -func (p *Progress) FileReport() string { - if p.fileReport == nil { - return "" - } - return p.fileReport.path -} diff --git a/internal/transfer/progress_counts.go b/internal/transfer/progress_counts.go deleted file mode 100644 index e7d8be12..00000000 --- a/internal/transfer/progress_counts.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -// ProgressCounts holds counts counted by Progress. -type ProgressCounts struct { - Failed, - Skipped, - Imported, - Exported, - Added, - Total uint -} - -// Progress returns ratio between processed messages (fully imported, skipped -// and failed ones) and total number of messages as percentage (0 - 1). -func (c *ProgressCounts) Progress() float32 { - progressed := c.Imported + c.Skipped + c.Failed - return float32(progressed) / float32(c.Total) -} diff --git a/internal/transfer/progress_test.go b/internal/transfer/progress_test.go deleted file mode 100644 index 899d7a5a..00000000 --- a/internal/transfer/progress_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "testing" - "time" - - "github.com/pkg/errors" - a "github.com/stretchr/testify/assert" - r "github.com/stretchr/testify/require" -) - -func TestProgressUpdateCount(t *testing.T) { - progress := newProgress(log, nil) - drainProgressUpdateChannel(&progress) - - progress.updateCount("inbox", 10) - progress.updateCount("archive", 20) - progress.updateCount("inbox", 12) - progress.updateCount("sent", 5) - progress.updateCount("foo", 4) - progress.updateCount("foo", 5) - - progress.finish() - - counts := progress.GetCounts() - r.Equal(t, uint(42), counts.Total) -} - -func TestProgressAddingMessages(t *testing.T) { - progress := newProgress(log, nil) - drainProgressUpdateChannel(&progress) - - // msg1 has no problem. - progress.addMessage("msg1", []string{}, []string{}) - progress.messageExported("msg1", []byte(""), nil) - progress.messageImported("msg1", "", nil) - - // msg2 has an import problem. - progress.addMessage("msg2", []string{}, []string{}) - progress.messageExported("msg2", []byte(""), nil) - progress.messageImported("msg2", "", errors.New("failed import")) - - // msg3 has an export problem. - progress.addMessage("msg3", []string{}, []string{}) - progress.messageExported("msg3", []byte(""), errors.New("failed export")) - - // msg4 has an export problem and import is also called. - progress.addMessage("msg4", []string{}, []string{}) - progress.messageExported("msg4", []byte(""), errors.New("failed export")) - progress.messageImported("msg4", "", nil) - - // msg5 is skipped. - progress.addMessage("msg5", []string{}, []string{}) - progress.messageSkipped("msg5") - - progress.finish() - - counts := progress.GetCounts() - a.Equal(t, uint(5), counts.Added) - a.Equal(t, uint(2), counts.Exported) - a.Equal(t, uint(2), counts.Imported) - a.Equal(t, uint(1), counts.Skipped) - a.Equal(t, uint(3), counts.Failed) - - errorsMap := map[string]string{} - for _, status := range progress.GetFailedMessages() { - errorsMap[status.SourceID] = status.GetErrorMessage() - } - a.Equal(t, map[string]string{ - "msg2": "failed to import: failed import", - "msg3": "failed to export: failed export", - "msg4": "failed to export: failed export", - }, errorsMap) -} - -func TestProgressFinish(t *testing.T) { - progress := newProgress(log, nil) - drainProgressUpdateChannel(&progress) - - progress.finish() - r.Nil(t, progress.updateCh) - - r.NotPanics(t, func() { progress.addMessage("msg", []string{}, []string{}) }) -} - -func TestProgressFatalError(t *testing.T) { - progress := newProgress(log, nil) - drainProgressUpdateChannel(&progress) - - progress.fatal(errors.New("fatal error")) - r.Nil(t, progress.updateCh) - - r.NotPanics(t, func() { progress.addMessage("msg", []string{}, []string{}) }) -} - -func TestFailUnpauseAndStops(t *testing.T) { - progress := newProgress(log, nil) - drainProgressUpdateChannel(&progress) - - progress.Pause("pausing") - progress.fatal(errors.New("fatal error")) - - r.Nil(t, progress.updateCh) - r.True(t, progress.isStopped) - r.False(t, progress.IsPaused()) - r.Eventually(t, progress.shouldStop, time.Second, 10*time.Millisecond) -} - -func TestStopClosesUpdates(t *testing.T) { - progress := newProgress(log, nil) - ch := progress.updateCh - - progress.Stop() - r.Nil(t, progress.updateCh) - r.PanicsWithError(t, "send on closed channel", func() { ch <- struct{}{} }) -} - -func drainProgressUpdateChannel(progress *Progress) { - // updateCh is not needed to drain under tests - timeout is implemented. - // But timeout takes time which would slow down tests. - go func() { - for range progress.updateCh { - } - }() -} diff --git a/internal/transfer/provider.go b/internal/transfer/provider.go deleted file mode 100644 index 25079914..00000000 --- a/internal/transfer/provider.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -// Provider provides interface for common operation with provider. -type Provider interface { - // ID is used for generating transfer ID by combining source and target ID. - ID() string - - // Mailboxes returns all available mailboxes. - // Provider used as source returns only non-empty maibloxes. - // Provider used as target does not return all mail maiblox. - Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) -} - -// SourceProvider provides interface of provider with support of export. -type SourceProvider interface { - Provider - - // TransferTo exports messages based on rules to channel. - TransferTo(transferRules, *Progress, chan<- Message) -} - -// TargetProvider provides interface of provider with support of import. -type TargetProvider interface { - Provider - - // DefaultMailboxes returns the default mailboxes for default rules if no other is found. - DefaultMailboxes(sourceMailbox Mailbox) (targetMailboxes []Mailbox) - - // CreateMailbox creates new mailbox to be used as target in transfer rules. - CreateMailbox(Mailbox) (Mailbox, error) - - // TransferFrom imports messages from channel. - TransferFrom(transferRules, *Progress, <-chan Message) -} diff --git a/internal/transfer/provider_eml.go b/internal/transfer/provider_eml.go deleted file mode 100644 index dbd2758f..00000000 --- a/internal/transfer/provider_eml.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -// EMLProvider implements import and export to/from EML file structure. -type EMLProvider struct { - root string -} - -// NewEMLProvider creates EMLProvider. -func NewEMLProvider(root string) *EMLProvider { - return &EMLProvider{ - root: root, - } -} - -// ID is used for generating transfer ID by combining source and target ID. -// We want to keep the same rules for import from or export to local files -// no matter exact path, therefore it returns constant. The same as EML. -func (p *EMLProvider) ID() string { - return "local" //nolint[goconst] -} - -// Mailboxes returns all available folder names from root of EML files. -// In case the same folder name is used more than once (for example root/a/foo -// and root/b/foo), it's treated as the same folder. -func (p *EMLProvider) Mailboxes(includeEmpty, includeAllMail bool) (mailboxes []Mailbox, err error) { - // Special case for exporting--we don't know the path before setup if finished. - if p.root == "" { - return - } - - var folderNames []string - if includeEmpty { - folderNames, err = getFolderNames(p.root) - } else { - folderNames, err = getFolderNamesWithFileSuffix(p.root, ".eml") - } - if err != nil { - return nil, err - } - - for _, folderName := range folderNames { - mailboxes = append(mailboxes, Mailbox{ - ID: "", - Name: folderName, - Color: "", - IsExclusive: false, - }) - } - - return mailboxes, nil -} diff --git a/internal/transfer/provider_eml_source.go b/internal/transfer/provider_eml_source.go deleted file mode 100644 index a31ac4ff..00000000 --- a/internal/transfer/provider_eml_source.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "io/ioutil" - "os" - "path/filepath" - - "github.com/pkg/errors" -) - -// TransferTo exports messages based on rules to channel. -func (p *EMLProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) { - log.Info("Started transfer from EML to channel") - defer log.Info("Finished transfer from EML to channel") - - filePathsPerFolder, err := p.getFilePathsPerFolder(rules) - if err != nil { - progress.fatal(err) - return - } - - if len(filePathsPerFolder) == 0 { - return - } - - // This list is not filtered by time but instead going throgh each file - // twice or keeping all in memory we will tell rough estimation which - // will be updated during processing each file. - for folderName, filePaths := range filePathsPerFolder { - if progress.shouldStop() { - break - } - - progress.updateCount(folderName, uint(len(filePaths))) - } - progress.countsFinal() - - for folderName, filePaths := range filePathsPerFolder { - // No error guaranteed by getFilePathsPerFolder. - rule, _ := rules.getRuleBySourceMailboxName(folderName) - log.WithField("rule", rule).Debug("Processing rule") - p.exportMessages(rule, filePaths, progress, ch) - } -} - -func (p *EMLProvider) getFilePathsPerFolder(rules transferRules) (map[string][]string, error) { - filePaths, err := getFilePathsWithSuffix(p.root, ".eml") - if err != nil { - return nil, err - } - - filePathsMap := map[string][]string{} - for _, filePath := range filePaths { - folder := filepath.Base(filepath.Dir(filepath.Join(p.root, filePath))) - _, err := rules.getRuleBySourceMailboxName(folder) - if err != nil { - log.WithField("msg", filePath).Trace("Message skipped due to folder name") - continue - } - - filePathsMap[folder] = append(filePathsMap[folder], filePath) - } - - return filePathsMap, nil -} - -func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *Progress, ch chan<- Message) { - for _, filePath := range filePaths { - if progress.shouldStop() { - break - } - - msg, err := p.exportMessage(rule, filePath) - - progress.addMessage(filePath, msg.sourceNames(), msg.targetNames()) - - // Read and check time in body only if the rule specifies it - // to not waste energy. - if err == nil && rule.HasTimeLimit() { - msgTime, msgTimeErr := getMessageTime(msg.Body) - if msgTimeErr != nil { - err = msgTimeErr - } else if !rule.isTimeInRange(msgTime) { - log.WithField("msg", filePath).Debug("Message skipped due to time") - progress.messageSkipped(filePath) - continue - } - } - - progress.messageExported(filePath, msg.Body, err) - if err == nil { - ch <- msg - } - } -} - -func (p *EMLProvider) exportMessage(rule *Rule, filePath string) (Message, error) { - fullFilePath := filepath.Clean(filepath.Join(p.root, filePath)) - file, err := os.Open(fullFilePath) //nolint[gosec] - if err != nil { - return Message{}, errors.Wrap(err, "failed to open message") - } - defer file.Close() //nolint[errcheck] - - body, err := ioutil.ReadAll(file) - if err != nil { - return Message{}, errors.Wrap(err, "failed to read message") - } - - return Message{ - ID: filePath, - Unread: false, - Body: body, - Sources: []Mailbox{rule.SourceMailbox}, - Targets: rule.TargetMailboxes, - }, nil -} diff --git a/internal/transfer/provider_eml_target.go b/internal/transfer/provider_eml_target.go deleted file mode 100644 index 6ef29b0c..00000000 --- a/internal/transfer/provider_eml_target.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "io/ioutil" - "os" - "path/filepath" - - "github.com/hashicorp/go-multierror" -) - -// DefaultMailboxes returns the default mailboxes for default rules if no other is found. -func (p *EMLProvider) DefaultMailboxes(sourceMailbox Mailbox) []Mailbox { - return []Mailbox{{ - Name: sourceMailbox.Name, - }} -} - -// CreateMailbox does nothing. Folders are created dynamically during the import. -func (p *EMLProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) { - return mailbox, nil -} - -// TransferFrom imports messages from channel. -func (p *EMLProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) { - log.Info("Started transfer from channel to EML") - defer log.Info("Finished transfer from channel to EML") - - err := p.createFolders(rules) - if err != nil { - progress.fatal(err) - return - } - - for msg := range ch { - for progress.shouldStop() { - break - } - - err := p.writeFile(msg) - progress.messageImported(msg.ID, "", err) - } -} - -func (p *EMLProvider) createFolders(rules transferRules) error { - for rule := range rules.iterateActiveRules() { - for _, mailbox := range rule.TargetMailboxes { - path := filepath.Join(p.root, sanitizeFileName(mailbox.Name)) - if err := os.MkdirAll(path, os.ModePerm); err != nil { - return err - } - } - } - return nil -} - -func (p *EMLProvider) writeFile(msg Message) error { - fileName := sanitizeFileName(filepath.Base(msg.ID)) - if filepath.Ext(fileName) != ".eml" { - fileName += ".eml" - } - - var err error - for _, mailbox := range msg.Targets { - path := filepath.Join(p.root, mailbox.Name, fileName) - - if localErr := ioutil.WriteFile(path, msg.Body, 0600); localErr != nil { - err = multierror.Append(err, localErr) - } - } - return err -} diff --git a/internal/transfer/provider_eml_test.go b/internal/transfer/provider_eml_test.go deleted file mode 100644 index 385d5f5d..00000000 --- a/internal/transfer/provider_eml_test.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "fmt" - "io/ioutil" - "os" - "testing" - "time" - - r "github.com/stretchr/testify/require" -) - -func newTestEMLProvider(path string) *EMLProvider { - if path == "" { - path = "testdata/eml" - } - return NewEMLProvider(path) -} - -func TestEMLProviderMailboxes(t *testing.T) { - provider := newTestEMLProvider("") - - tests := []struct { - includeEmpty bool - wantMailboxes []Mailbox - }{ - {true, []Mailbox{ - {Name: "Foo"}, - {Name: "Inbox"}, - {Name: "eml"}, - }}, - {false, []Mailbox{ - {Name: "Foo"}, - {Name: "Inbox"}, - }}, - } - for _, tc := range tests { - tc := tc - t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) { - mailboxes, err := provider.Mailboxes(tc.includeEmpty, false) - r.NoError(t, err) - r.Equal(t, tc.wantMailboxes, mailboxes) - }) - } -} - -func TestEMLProviderTransferTo(t *testing.T) { - provider := newTestEMLProvider("") - - rules, rulesClose := newTestRules(t) - defer rulesClose() - setupEMLRules(rules) - - testTransferTo(t, rules, provider, []string{ - "Foo/msg.eml", - "Inbox/msg.eml", - }) -} - -func TestEMLProviderTransferFrom(t *testing.T) { - dir, err := ioutil.TempDir("", "eml") - r.NoError(t, err) - defer os.RemoveAll(dir) //nolint[errcheck] - - provider := newTestEMLProvider(dir) - - rules, rulesClose := newTestRules(t) - defer rulesClose() - setupEMLRules(rules) - - testTransferFrom(t, rules, provider, []Message{ - {ID: "Foo/msg.eml", Body: getTestMsgBody("msg"), Targets: []Mailbox{{Name: "Foo"}}}, - }) - - checkEMLFileStructure(t, dir, []string{ - "Foo/msg.eml", - }) -} - -func TestEMLProviderTransferFromTo(t *testing.T) { - dir, err := ioutil.TempDir("", "eml") - r.NoError(t, err) - defer os.RemoveAll(dir) //nolint[errcheck] - - source := newTestEMLProvider("") - target := newTestEMLProvider(dir) - - rules, rulesClose := newTestRules(t) - defer rulesClose() - setupEMLRules(rules) - - testTransferFromTo(t, rules, source, target, 5*time.Second) - - checkEMLFileStructure(t, dir, []string{ - "Foo/msg.eml", - "Inbox/msg.eml", - }) -} - -func setupEMLRules(rules transferRules) { - _ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0) - _ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0) -} - -func checkEMLFileStructure(t *testing.T, root string, expectedFiles []string) { - files, err := getFilePathsWithSuffix(root, ".eml") - r.NoError(t, err) - r.Equal(t, expectedFiles, files) -} diff --git a/internal/transfer/provider_imap.go b/internal/transfer/provider_imap.go deleted file mode 100644 index c88dc8fe..00000000 --- a/internal/transfer/provider_imap.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "net" - "strings" - - "github.com/emersion/go-imap" - "github.com/emersion/go-sasl" -) - -type IMAPClientProvider interface { - Capability() (map[string]bool, error) - Support(capability string) (bool, error) - State() imap.ConnState - SupportAuth(mech string) (bool, error) - Authenticate(auth sasl.Client) error - Login(username, password string) error - List(ref, name string, ch chan *imap.MailboxInfo) error - Select(name string, readOnly bool) (*imap.MailboxStatus, error) - Fetch(seqset *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) error - UidFetch(seqset *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) error -} - -// IMAPProvider implements export from IMAP server. -type IMAPProvider struct { - username string - password string - addr string - - clientDialer func(addr string) (IMAPClientProvider, error) - client IMAPClientProvider - - timeIt *timeIt -} - -// NewIMAPProvider returns new IMAPProvider. -func NewIMAPProvider(username, password, host, port string) (*IMAPProvider, error) { - return newIMAPProvider(imapClientDial, username, password, host, port) -} - -func newIMAPProvider(clientDialer func(string) (IMAPClientProvider, error), username, password, host, port string) (*IMAPProvider, error) { - p := &IMAPProvider{ - username: username, - password: password, - addr: net.JoinHostPort(host, port), - - timeIt: newTimeIt("imap"), - - clientDialer: clientDialer, - } - - if err := p.auth(); err != nil { - return nil, err - } - - return p, nil -} - -// ID is used for generating transfer ID by combining source and target ID. -// We want to keep the same rules for import from any IMAP server, therefore -// it returns constant. -func (p *IMAPProvider) ID() string { - return "imap" -} - -// Mailboxes returns all available folder names from root of EML files. -// In case the same folder name is used more than once (for example root/a/foo -// and root/b/foo), it's treated as the same folder. -func (p *IMAPProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) { - mailboxesInfo, err := p.list() - if err != nil { - return nil, err - } - - mailboxes := []Mailbox{} - for _, mailbox := range mailboxesInfo { - hasNoSelect := false - for _, attrib := range mailbox.Attributes { - if strings.ToLower(attrib) == "\\noselect" { - hasNoSelect = true - break - } - } - if hasNoSelect { - continue - } - - if !includeEmpty || true { - mailboxStatus, err := p.selectIn(mailbox.Name) - if err != nil { - return nil, err - } - if mailboxStatus.Messages == 0 { - continue - } - } - - mailboxes = append(mailboxes, Mailbox{ - ID: "", - Name: mailbox.Name, - Color: "", - IsExclusive: false, - }) - } - return mailboxes, nil -} diff --git a/internal/transfer/provider_imap_errors.go b/internal/transfer/provider_imap_errors.go deleted file mode 100644 index 795b9cd4..00000000 --- a/internal/transfer/provider_imap_errors.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -// imapError is base for all IMAP errors. -type imapError struct { - Message string - Err error -} - -func (e imapError) Error() string { - return e.Message + ": " + e.Err.Error() -} - -func (e imapError) Unwrap() error { - return e.Err -} - -func (e imapError) Cause() error { - return e.Err -} - -// ErrIMAPConnection is error representing connection issues. -type ErrIMAPConnection struct { - imapError -} - -func (e ErrIMAPConnection) Is(target error) bool { - _, ok := target.(*ErrIMAPConnection) - return ok -} - -// ErrIMAPAuth is error representing authentication issues. -type ErrIMAPAuth struct { - imapError -} - -func (e ErrIMAPAuth) Is(target error) bool { - _, ok := target.(*ErrIMAPAuth) - return ok -} - -// ErrIMAPAuthMethod is error representing wrong auth method. -type ErrIMAPAuthMethod struct { - imapError -} - -func (e ErrIMAPAuthMethod) Is(target error) bool { - _, ok := target.(*ErrIMAPAuthMethod) - return ok -} diff --git a/internal/transfer/provider_imap_source.go b/internal/transfer/provider_imap_source.go deleted file mode 100644 index f81c77b4..00000000 --- a/internal/transfer/provider_imap_source.go +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "fmt" - "io/ioutil" - - "github.com/emersion/go-imap" -) - -type imapMessageInfo struct { - id string - uid uint32 - size uint32 -} - -const ( - imapPageSize = uint32(2000) // Optimized on Gmail. - imapMaxFetchSize = uint32(50 * 1000 * 1000) // Size in octets. If 0, it will use one fetch per message. -) - -// TransferTo exports messages based on rules to channel. -func (p *IMAPProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) { - log.Info("Started transfer from IMAP to channel") - defer log.Info("Finished transfer from IMAP to channel") - - p.timeIt.clear() - defer p.timeIt.logResults() - - imapMessageInfoMap := p.loadMessageInfoMap(rules, progress) - - for rule := range rules.iterateActiveRules() { - messagesInfo := imapMessageInfoMap[rule.SourceMailbox.Name] - if messagesInfo == nil { - log.WithField("rule", rule).Warn("Rule has no message info") - continue - } - log.WithField("rule", rule).Debug("Processing rule") - p.transferTo(rule, messagesInfo, progress, ch) - } -} - -func (p *IMAPProvider) loadMessageInfoMap(rules transferRules, progress *Progress) map[string]map[string]imapMessageInfo { - res := map[string]map[string]imapMessageInfo{} - - for rule := range rules.iterateActiveRules() { - if progress.shouldStop() { - break - } - - mailboxName := rule.SourceMailbox.Name - var mailbox *imap.MailboxStatus - progress.callWrap(func() error { - var err error - mailbox, err = p.selectIn(mailboxName) - return err - }) - if mailbox == nil { - log.WithField("rule", rule.SourceMailbox.Name).Warn("Failed to select into mailbox") - continue - } - if mailbox.Messages == 0 { - continue - } - - messagesInfo := p.loadMessagesInfo(rule, progress, mailbox.UidValidity, mailbox.Messages) - res[rule.SourceMailbox.Name] = messagesInfo - progress.updateCount(rule.SourceMailbox.Name, uint(len(messagesInfo))) - } - progress.countsFinal() - - return res -} - -func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValidity, count uint32) map[string]imapMessageInfo { - p.timeIt.start("load", rule.SourceMailbox.Name) - defer p.timeIt.stop("load", rule.SourceMailbox.Name) - - log := log.WithField("mailbox", rule.SourceMailbox.Name) - messagesInfo := map[string]imapMessageInfo{} - - fetchItems := []imap.FetchItem{imap.FetchUid, imap.FetchRFC822Size} - if rule.HasTimeLimit() { - fetchItems = append(fetchItems, imap.FetchEnvelope) - } - - processMessageCallback := func(imapMessage *imap.Message) { - if rule.HasTimeLimit() { - t := imapMessage.Envelope.Date.Unix() - if t != 0 && !rule.isTimeInRange(t) { - log.WithField("uid", imapMessage.Uid).Debug("Message skipped due to time") - return - } - } - id := getUniqueMessageID(rule.SourceMailbox.Name, uidValidity, imapMessage.Uid) - // We use ID as key to ensure we have every unique message only once. - // Some IMAP servers responded twice the same message... - messagesInfo[id] = imapMessageInfo{ - id: id, - uid: imapMessage.Uid, - size: imapMessage.Size, - } - progress.addMessage(id, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames()) - } - - pageStart := uint32(1) - pageEnd := imapPageSize - for { - if progress.shouldStop() || pageStart > count { - break - } - - // Some servers do not accept message sequence number higher than the total count. - if pageEnd > count { - pageEnd = count - } - - seqSet := &imap.SeqSet{} - seqSet.AddRange(pageStart, pageEnd) - err := p.fetch(rule.SourceMailbox.Name, seqSet, fetchItems, processMessageCallback) - if err != nil { - log.WithError(err).WithField("idx", seqSet).Warning("Load batch fetch failed, trying one by one") - for ; pageStart <= pageEnd; pageStart++ { - seqSet := &imap.SeqSet{} - seqSet.AddNum(pageStart) - if err := p.fetch(rule.SourceMailbox.Name, seqSet, fetchItems, processMessageCallback); err != nil { - log.WithError(err).WithField("idx", seqSet).Warning("Load fetch failed, skipping the message") - } - } - } - - pageStart = pageEnd + 1 - pageEnd += imapPageSize - } - return messagesInfo -} - -func (p *IMAPProvider) transferTo(rule *Rule, messagesInfo map[string]imapMessageInfo, progress *Progress, ch chan<- Message) { - progress.callWrap(func() error { - _, err := p.selectIn(rule.SourceMailbox.Name) - return err - }) - - seqSet := &imap.SeqSet{} - seqSetSize := uint32(0) - uidToID := map[uint32]string{} - - for _, messageInfo := range messagesInfo { - if progress.shouldStop() { - break - } - - if seqSetSize != 0 && (seqSetSize+messageInfo.size) > imapMaxFetchSize { - log.WithField("mailbox", rule.SourceMailbox.Name).WithField("seq", seqSet).WithField("size", seqSetSize).Debug("Fetching messages") - p.exportMessages(rule, progress, ch, seqSet, uidToID) - - seqSet = &imap.SeqSet{} - seqSetSize = 0 - uidToID = map[uint32]string{} - } - - seqSet.AddNum(messageInfo.uid) - seqSetSize += messageInfo.size - uidToID[messageInfo.uid] = messageInfo.id - } - - if len(uidToID) != 0 { - log.WithField("mailbox", rule.SourceMailbox.Name).WithField("seq", seqSet).WithField("size", seqSetSize).Debug("Fetching messages") - p.exportMessages(rule, progress, ch, seqSet, uidToID) - } -} - -func (p *IMAPProvider) exportMessages(rule *Rule, progress *Progress, ch chan<- Message, seqSet *imap.SeqSet, uidToID map[uint32]string) { - section := &imap.BodySectionName{} - items := []imap.FetchItem{imap.FetchUid, imap.FetchFlags, section.FetchItem()} - - processMessageCallback := func(imapMessage *imap.Message) { - if progress.shouldStop() { - return - } - - id, ok := uidToID[imapMessage.Uid] - - // Sometimes, server sends not requested messages. - if !ok { - log.WithField("uid", imapMessage.Uid).Warning("Message skipped: not requested") - return - } - - // Sometimes, server sends message twice, once with body and once without it. - bodyReader := imapMessage.GetBody(section) - if bodyReader == nil { - log.WithField("uid", imapMessage.Uid).Warning("Message skipped: no body") - return - } - - body, err := ioutil.ReadAll(bodyReader) - progress.messageExported(id, body, err) - if err == nil { - msg := p.exportMessage(rule, id, imapMessage, body) - - p.timeIt.stop("fetch", rule.SourceMailbox.Name) - ch <- msg - p.timeIt.start("fetch", rule.SourceMailbox.Name) - } - } - - p.timeIt.start("fetch", rule.SourceMailbox.Name) - progress.callWrap(func() error { - return p.uidFetch(rule.SourceMailbox.Name, seqSet, items, processMessageCallback) - }) - p.timeIt.stop("fetch", rule.SourceMailbox.Name) -} - -func (p *IMAPProvider) exportMessage(rule *Rule, id string, imapMessage *imap.Message, body []byte) Message { - unread := true - for _, flag := range imapMessage.Flags { - if flag == imap.SeenFlag { - unread = false - } - } - - return Message{ - ID: id, - Unread: unread, - Body: body, - Sources: []Mailbox{rule.SourceMailbox}, - Targets: rule.TargetMailboxes, - } -} - -func getUniqueMessageID(mailboxName string, uidValidity, uid uint32) string { - return fmt.Sprintf("%s_%d:%d", mailboxName, uidValidity, uid) -} diff --git a/internal/transfer/provider_imap_test.go b/internal/transfer/provider_imap_test.go deleted file mode 100644 index 245d30c3..00000000 --- a/internal/transfer/provider_imap_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "fmt" - "testing" - - "github.com/emersion/go-imap" - gomock "github.com/golang/mock/gomock" - "github.com/pkg/errors" - r "github.com/stretchr/testify/require" -) - -func newTestIMAPProvider(t *testing.T, m mocks) *IMAPProvider { - m.imapClientProvider.EXPECT().State().Return(imap.ConnectedState).AnyTimes() - m.imapClientProvider.EXPECT().Capability().Return(map[string]bool{ - "AUTH": true, - }, nil).AnyTimes() - - dialer := func(string) (IMAPClientProvider, error) { - return m.imapClientProvider, nil - } - provider, err := newIMAPProvider(dialer, "user", "pass", "host", "42") - r.NoError(t, err) - return provider -} - -func TestProviderIMAPLoadMessagesInfo(t *testing.T) { - m := initMocks(t) - defer m.ctrl.Finish() - - provider := newTestIMAPProvider(t, m) - - progress := newProgress(log, nil) - drainProgressUpdateChannel(&progress) - - rule := &Rule{SourceMailbox: Mailbox{Name: "Mailbox"}} - uidValidity := 1 - count := 2200 - failingIndex := 2100 - - m.imapClientProvider.EXPECT().Select(rule.SourceMailbox.Name, gomock.Any()).Return(&imap.MailboxStatus{}, nil).AnyTimes() - m.imapClientProvider.EXPECT(). - Fetch(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(seqSet *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) error { - defer close(ch) - for _, seq := range seqSet.Set { - for i := seq.Start; i <= seq.Stop; i++ { - if int(i) == failingIndex { - return errors.New("internal server error") - } - ch <- &imap.Message{ - SeqNum: i, - Uid: i * 10, - Size: i * 100, - } - } - } - return nil - }). - // 2200 messages is split into two batches (2000 and 200), - // the second one fails and makes 200 calls (one-by-one). - // Plus two failed requests are repeated `imapRetries` times. - Times(2 + 200 + (2 * (imapRetries - 1))) - - messageInfo := provider.loadMessagesInfo(rule, &progress, uint32(uidValidity), uint32(count)) - - r.Equal(t, count-1, len(messageInfo)) // One message produces internal server error. - for index := 1; index <= count; index++ { - uid := index * 10 - key := fmt.Sprintf("%s_%d:%d", rule.SourceMailbox.Name, uidValidity, uid) - - if index == failingIndex { - r.Empty(t, messageInfo[key]) - continue - } - - r.Equal(t, imapMessageInfo{ - id: key, - uid: uint32(uid), - size: uint32(index * 100), - }, messageInfo[key]) - } -} diff --git a/internal/transfer/provider_imap_utils.go b/internal/transfer/provider_imap_utils.go deleted file mode 100644 index 0ac8a8f0..00000000 --- a/internal/transfer/provider_imap_utils.go +++ /dev/null @@ -1,311 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "crypto/tls" - "fmt" - "net" - "net/http" - "strings" - "time" - - imapID "github.com/ProtonMail/go-imap-id" - "github.com/ProtonMail/proton-bridge/internal/constants" - "github.com/emersion/go-imap" - imapClient "github.com/emersion/go-imap/client" - "github.com/emersion/go-sasl" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -const ( - imapDialTimeout = 5 * time.Second - imapRetries = 10 - imapReconnectTimeout = 30 * time.Minute - imapReconnectSleep = time.Minute - - protonStatusURL = "http://protonstatus.com/vpn_status" -) - -type imapErrorLogger struct { - log *logrus.Entry -} - -func (l *imapErrorLogger) Printf(f string, v ...interface{}) { - l.log.Errorf(f, v...) -} - -func (l *imapErrorLogger) Println(v ...interface{}) { - l.log.Errorln(v...) -} - -func imapClientDial(addr string) (IMAPClientProvider, error) { - if _, err := net.DialTimeout("tcp", addr, imapDialTimeout); err != nil { - return nil, errors.Wrap(err, "failed to dial server") - } - - client, err := imapClientDialHelper(addr) - if err == nil { - client.ErrorLog = &imapErrorLogger{logrus.WithField("pkg", "imap-client")} - // Logrus `WriterLevel` fails for big messages because of bufio.MaxScanTokenSize limit. - // Also, this spams a lot, uncomment once needed during development. - // client.SetDebug(imap.NewDebugWriter( - // logrus.WithField("pkg", "imap/client").WriterLevel(logrus.TraceLevel), - // logrus.WithField("pkg", "imap/server").WriterLevel(logrus.TraceLevel), - // )) - } - return client, err -} - -func imapClientDialHelper(addr string) (*imapClient.Client, error) { - host, _, _ := net.SplitHostPort(addr) - if host == "127.0.0.1" { - return imapClient.Dial(addr) - } - - // IMAP mail.yahoo.com has problem with golang TLS 1.3 implementation - // with weird behaviour, i.e., Yahoo does not return error during dial - // or handshake but server does logs out right after successful login - // leaving no time to perform any action. - // Limiting TLS to version 1.2 is working just fine. - var tlsConf *tls.Config - if strings.Contains(strings.ToLower(host), "yahoo") { - log.Warning("Yahoo server detected: limiting maximal TLS version to 1.2.") - tlsConf = &tls.Config{MaxVersion: tls.VersionTLS12} //nolint[gosec] G402 - } - return imapClient.DialTLS(addr, tlsConf) -} - -func (p *IMAPProvider) ensureConnection(callback func() error) error { - return p.ensureConnectionAndSelection(callback, "") -} - -func (p *IMAPProvider) ensureConnectionAndSelection(callback func() error, ensureSelectedIn string) error { - var callErr error - for i := 1; i <= imapRetries; i++ { - callErr = callback() - if callErr == nil { - return nil - } - - log.WithField("attempt", i).WithError(callErr).Warning("IMAP call failed, trying reconnect") - err := p.tryReconnect(ensureSelectedIn) - if err != nil { - return err - } - } - return errors.Wrap(callErr, "too many retries") -} - -func (p *IMAPProvider) tryReconnect(ensureSelectedIn string) error { - start := time.Now() - var previousErr error - for { - if time.Since(start) > imapReconnectTimeout { - return previousErr - } - - err := checkConnection() - log.WithError(err).Debug("Connection check") - if err != nil { - time.Sleep(imapReconnectSleep) - previousErr = err - continue - } - - err = p.reauth() - log.WithError(err).Debug("Reauth") - if err != nil { - time.Sleep(imapReconnectSleep) - previousErr = err - continue - } - - if ensureSelectedIn != "" { - _, err = p.client.Select(ensureSelectedIn, true) - log.WithError(err).Debug("Reselect") - if err != nil { - previousErr = err - continue - } - } - - break - } - return nil -} - -func (p *IMAPProvider) reauth() error { - var state imap.ConnState - - // In some cases once go-imap fails, we cannot issue another command - // because it would dead-lock. Let's simply ignore it, we want to open - // new connection anyway. - ch := make(chan struct{}) - go func() { - defer close(ch) - if _, err := p.client.Capability(); err != nil { - state = p.client.State() - } - }() - select { - case <-ch: - case <-time.After(30 * time.Second): - } - - log.WithField("addr", p.addr).WithField("state", state).Debug("Reconnecting") - p.client = nil - return p.auth() -} - -func (p *IMAPProvider) auth() error { //nolint[funlen] - log := log.WithField("addr", p.addr) - - log.Info("Connecting to server") - - client, err := p.clientDialer(p.addr) - if err != nil { - return ErrIMAPConnection{imapError{Err: err, Message: "failed to connect to server"}} - } - p.client = client - - log.Info("Connected") - - if (p.client.State() & imap.AuthenticatedState) == imap.AuthenticatedState { - return nil - } - - capability, err := p.client.Capability() - log.WithField("capability", capability).WithError(err).Debug("Server capability") - if err != nil { - return ErrIMAPConnection{imapError{Err: err, Message: "failed to get capabilities"}} - } - - // SASL AUTH PLAIN - if ok, _ := p.client.SupportAuth("PLAIN"); p.client.State() == imap.NotAuthenticatedState && ok { - log.Debug("Trying plain auth") - authPlain := sasl.NewPlainClient("", p.username, p.password) - if err = p.client.Authenticate(authPlain); err != nil { - return ErrIMAPAuth{imapError{Err: err, Message: "plain auth failed"}} - } - } - - // LOGIN: if the server reports the IMAP4rev1 capability then it is standards conformant and must support login. - if ok, _ := p.client.Support("IMAP4rev1"); p.client.State() == imap.NotAuthenticatedState && ok { - log.Debug("Trying login") - if err = p.client.Login(p.username, p.password); err != nil { - return ErrIMAPAuth{imapError{Err: err, Message: "login failed"}} - } - } - - if p.client.State() == imap.NotAuthenticatedState { - return ErrIMAPAuthMethod{imapError{Err: err, Message: "unknown auth method"}} - } - - log.Info("Logged in") - - if c, ok := p.client.(*imapClient.Client); ok { - idClient := imapID.NewClient(c) - if ok, err := idClient.SupportID(); err == nil && ok { - serverID, err := idClient.ID(imapID.ID{ - imapID.FieldName: "ImportExport", - imapID.FieldVersion: constants.Version, - }) - log.WithField("ID", serverID).WithError(err).Debug("Server info") - } - } - - return err -} - -func (p *IMAPProvider) list() (mailboxes []*imap.MailboxInfo, err error) { - err = p.ensureConnection(func() error { - mailboxesCh := make(chan *imap.MailboxInfo) - doneCh := make(chan error) - - go func() { - doneCh <- p.client.List("", "*", mailboxesCh) - }() - - for mailbox := range mailboxesCh { - mailboxes = append(mailboxes, mailbox) - } - - return <-doneCh - }) - return -} - -func (p *IMAPProvider) selectIn(mailboxName string) (mailbox *imap.MailboxStatus, err error) { - err = p.ensureConnection(func() error { - mailbox, err = p.client.Select(mailboxName, true) - return err - }) - return -} - -func (p *IMAPProvider) fetch(ensureSelectedIn string, seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error { - return p.fetchHelper(false, ensureSelectedIn, seqSet, items, processMessageCallback) -} - -func (p *IMAPProvider) uidFetch(ensureSelectedIn string, seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error { - return p.fetchHelper(true, ensureSelectedIn, seqSet, items, processMessageCallback) -} - -func (p *IMAPProvider) fetchHelper(uid bool, ensureSelectedIn string, seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error { - return p.ensureConnectionAndSelection(func() error { - messagesCh := make(chan *imap.Message) - doneCh := make(chan error) - - go func() { - if uid { - doneCh <- p.client.UidFetch(seqSet, items, messagesCh) - } else { - doneCh <- p.client.Fetch(seqSet, items, messagesCh) - } - }() - - for message := range messagesCh { - processMessageCallback(message) - } - - err := <-doneCh - return err - }, ensureSelectedIn) -} - -// checkConnection returns an error if there is no internet connection. -// Note we don't want to use client manager because it only reports connection -// issues with API; we are only interested here whether we can reach -// third-party IMAP servers. -func checkConnection() error { - client := &http.Client{Timeout: time.Second * 10} - - resp, err := client.Get(protonStatusURL) - if err != nil { - return err - } - - _ = resp.Body.Close() - if resp.StatusCode != 200 { - return fmt.Errorf("HTTP status code %d", resp.StatusCode) - } - - return nil -} diff --git a/internal/transfer/provider_local.go b/internal/transfer/provider_local.go deleted file mode 100644 index dcd5a303..00000000 --- a/internal/transfer/provider_local.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -// LocalProvider implements import from local EML and MBOX file structure. -type LocalProvider struct { - root string - emlProvider *EMLProvider - mboxProvider *MBOXProvider -} - -func NewLocalProvider(root string) *LocalProvider { - return &LocalProvider{ - root: root, - emlProvider: NewEMLProvider(root), - mboxProvider: NewMBOXProvider(root), - } -} - -// ID is used for generating transfer ID by combining source and target ID. -// We want to keep the same rules for import from or export to local files -// no matter exact path, therefore it returns constant. -// The same as EML and MBOX. -func (p *LocalProvider) ID() string { - return "local" //nolint[goconst] -} - -// Mailboxes returns all available folder names from root of EML and MBOX files. -func (p *LocalProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) { - mailboxes, err := p.emlProvider.Mailboxes(includeEmpty, includeAllMail) - if err != nil { - return nil, err - } - - mboxMailboxes, err := p.mboxProvider.Mailboxes(includeEmpty, includeAllMail) - if err != nil { - return nil, err - } - - for _, mboxMailbox := range mboxMailboxes { - found := false - for _, mailboxes := range mailboxes { - if mboxMailbox.Name == mailboxes.Name { - found = true - break - } - } - if !found { - mailboxes = append(mailboxes, mboxMailbox) - } - } - return mailboxes, nil -} diff --git a/internal/transfer/provider_local_source.go b/internal/transfer/provider_local_source.go deleted file mode 100644 index 65b9852c..00000000 --- a/internal/transfer/provider_local_source.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "sync" -) - -// TransferTo exports messages based on rules to channel. -func (p *LocalProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) { - log.Info("Started transfer from EML and MBOX to channel") - defer log.Info("Finished transfer from EML and MBOX to channel") - - var wg sync.WaitGroup - wg.Add(2) - - go func() { - defer wg.Done() - p.emlProvider.TransferTo(rules, progress, ch) - }() - go func() { - defer wg.Done() - p.mboxProvider.TransferTo(rules, progress, ch) - }() - - wg.Wait() -} diff --git a/internal/transfer/provider_local_test.go b/internal/transfer/provider_local_test.go deleted file mode 100644 index 3c60edd2..00000000 --- a/internal/transfer/provider_local_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "fmt" - "testing" - - r "github.com/stretchr/testify/require" -) - -func newTestLocalProvider(path string) *LocalProvider { - if path == "" { - path = "testdata/emlmbox" - } - return NewLocalProvider(path) -} - -func TestLocalProviderMailboxes(t *testing.T) { - provider := newTestLocalProvider("") - - tests := []struct { - includeEmpty bool - wantMailboxes []Mailbox - }{ - {true, []Mailbox{ - {Name: "Foo"}, - {Name: "emlmbox"}, - {Name: "Inbox"}, - }}, - {false, []Mailbox{ - {Name: "Foo"}, - {Name: "Inbox"}, - }}, - } - for _, tc := range tests { - tc := tc - t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) { - mailboxes, err := provider.Mailboxes(tc.includeEmpty, false) - r.NoError(t, err) - r.Equal(t, tc.wantMailboxes, mailboxes) - }) - } -} - -func TestLocalProviderTransferTo(t *testing.T) { - provider := newTestLocalProvider("") - - rules, rulesClose := newTestRules(t) - defer rulesClose() - setupEMLMBOXRules(rules) - - testTransferTo(t, rules, provider, []string{ - "Foo/msg.eml", - "Inbox.mbox:1", - }) -} - -func setupEMLMBOXRules(rules transferRules) { - _ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0) - _ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0) -} diff --git a/internal/transfer/provider_mbox.go b/internal/transfer/provider_mbox.go deleted file mode 100644 index 16c24727..00000000 --- a/internal/transfer/provider_mbox.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" -) - -// MBOXProvider implements import and export to/from MBOX structure. -type MBOXProvider struct { - root string -} - -func NewMBOXProvider(root string) *MBOXProvider { - return &MBOXProvider{ - root: root, - } -} - -// ID is used for generating transfer ID by combining source and target ID. -// We want to keep the same rules for import from or export to local files -// no matter exact path, therefore it returns constant. The same as EML. -func (p *MBOXProvider) ID() string { - return "local" //nolint[goconst] -} - -// Mailboxes returns all available folder names from root of EML files. -// In case the same folder name is used more than once (for example root/a/foo -// and root/b/foo), it's treated as the same folder. -func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) { - filePaths, err := getAllPathsWithSuffix(p.root, ".mbox") - if err != nil { - return nil, err - } - - mailboxNames := map[string]bool{} - for _, filePath := range filePaths { - fileName := filepath.Base(filePath) - filePath, err := p.handleAppleMailMBOXStructure(filePath) - if err != nil { - log.WithError(err).Warn("Failed to handle MBOX structure") - continue - } - - mailboxName := strings.TrimSuffix(fileName, ".mbox") - mailboxNames[mailboxName] = true - - labels, err := getGmailLabelsFromMboxFile(filepath.Join(p.root, filePath)) - if err != nil { - log.WithError(err).Error("Failed to get gmail labels from mbox file") - continue - } - for label := range labels { - mailboxNames[label] = true - } - } - - mailboxes := []Mailbox{} - for mailboxName := range mailboxNames { - mailboxes = append(mailboxes, Mailbox{ - ID: "", - Name: mailboxName, - Color: "", - IsExclusive: false, - }) - } - return mailboxes, nil -} - -// handleAppleMailMBOXStructure changes the path of mailbox directory to -// the path of mbox file. Apple Mail MBOX exports has this structure: -// `Folder.mbox` directory with `mbox` file inside. -// Example: `Folder.mbox/mbox` (and this function converts `Folder.mbox` -// to `Folder.mbox/mbox`). -func (p *MBOXProvider) handleAppleMailMBOXStructure(filePath string) (string, error) { - if info, err := os.Stat(filepath.Join(p.root, filePath)); err == nil && info.IsDir() { - if _, err := os.Stat(filepath.Join(p.root, filePath, "mbox")); err != nil { - return "", errors.Wrap(err, "wrong mbox structure") - } - return filepath.Join(filePath, "mbox"), nil - } - return filePath, nil -} diff --git a/internal/transfer/provider_mbox_gmail_labels.go b/internal/transfer/provider_mbox_gmail_labels.go deleted file mode 100644 index 243c16cf..00000000 --- a/internal/transfer/provider_mbox_gmail_labels.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "bufio" - "bytes" - "io" - "mime" - "os" - "strings" -) - -type stringSet map[string]bool - -const xGmailLabelsHeader = "X-Gmail-Labels" - -// filteredOutGmailLabels is set of labels which we don't want to show to users -// as they might be auto-generated by Gmail and unwanted. -var filteredOutGmailLabels = []string{ //nolint[gochecknoglobals] - "Unread", - "Opened", - "IMAP_Junk", - "IMAP_NonJunk", - "IMAP_NotJunk", - "IMAP_$NotJunk", -} - -func getGmailLabelsFromMboxFile(filePath string) (stringSet, error) { - f, err := os.Open(filePath) //nolint[gosec] - if err != nil { - return nil, err - } - return getGmailLabelsFromMboxReader(f) -} - -func getGmailLabelsFromMboxReader(f io.Reader) (stringSet, error) { - allLabels := stringSet{} - - // Scanner is not used as it does not support long lines and some mbox - // files contain very long lines even though that should not be happening. - r := bufio.NewReader(f) - for { - b, isPrefix, err := r.ReadLine() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - for isPrefix { - _, isPrefix, err = r.ReadLine() - if err != nil { - break - } - } - if bytes.HasPrefix(b, []byte(xGmailLabelsHeader)) { - for label := range getGmailLabelsFromValue(string(b)) { - allLabels[label] = true - } - } - } - - return allLabels, nil -} - -func getGmailLabelsFromMessage(body []byte) (stringSet, error) { - header, err := getMessageHeader(body) - if err != nil { - return nil, err - } - labels := header.Get(xGmailLabelsHeader) - return getGmailLabelsFromValue(labels), nil -} - -func getGmailLabelsFromValue(value string) stringSet { - value = strings.TrimPrefix(value, xGmailLabelsHeader+":") - if decoded, err := new(mime.WordDecoder).DecodeHeader(value); err != nil { - log.WithError(err).Error("Failed to decode header") - } else { - value = decoded - } - - labels := stringSet{} - for _, label := range strings.Split(value, ",") { - label = strings.TrimSpace(label) - if label == "" { - continue - } - skip := false - for _, filteredOutLabel := range filteredOutGmailLabels { - if label == filteredOutLabel { - skip = true - break - } - } - if skip { - continue - } - labels[label] = true - } - return labels -} diff --git a/internal/transfer/provider_mbox_gmail_labels_test.go b/internal/transfer/provider_mbox_gmail_labels_test.go deleted file mode 100644 index 293b10a2..00000000 --- a/internal/transfer/provider_mbox_gmail_labels_test.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "fmt" - "strings" - "testing" - - r "github.com/stretchr/testify/require" -) - -func TestGetGmailLabelsFromMboxReader(t *testing.T) { - mboxFile := `From - Mon May 4 16:40:31 2020 -Subject: Test 1 -X-Gmail-Labels: Foo,Bar - -hello - -From - Mon May 4 16:40:31 2020 -Subject: Test 2 -X-Gmail-Labels: Foo , Baz - -hello - -From - Mon May 4 16:40:31 2020 -Subject: Test 3 -X-Gmail-Labels: , - -hello - -From - Mon May 4 16:40:31 2020 -Subject: Test 4 -X-Gmail-Labels: - -hello - -From - Mon May 4 16:40:31 2020 -Subject: Test 5 - -hello - -` - mboxReader := strings.NewReader(mboxFile) - labels, err := getGmailLabelsFromMboxReader(mboxReader) - r.NoError(t, err) - r.Equal(t, toSet("Foo", "Bar", "Baz"), labels) -} - -func TestGetGmailLabelsFromMessage(t *testing.T) { - tests := []struct { - body string - wantLabels stringSet - }{ - {`Subject: One -X-Gmail-Labels: Foo,Bar - -Hello -`, toSet("Foo", "Bar")}, - {`Subject: Two -X-Gmail-Labels: Foo , Bar , - -Hello -`, toSet("Foo", "Bar")}, - {`Subject: Three -X-Gmail-Labels: , - -Hello -`, toSet()}, - {`Subject: Four -X-Gmail-Labels: - -Hello -`, toSet()}, - {`Subject: Five - -Hello -`, toSet()}, - } - for _, tc := range tests { - tc := tc - t.Run(fmt.Sprintf("%v", tc.body), func(t *testing.T) { - labels, err := getGmailLabelsFromMessage([]byte(tc.body)) - r.NoError(t, err) - r.Equal(t, tc.wantLabels, labels) - }) - } -} - -func TestGetGmailLabelsFromValue(t *testing.T) { - tests := []struct { - value string - wantLabels stringSet - }{ - {"Foo,Bar", toSet("Foo", "Bar")}, - {" Foo , Bar ", toSet("Foo", "Bar")}, - {" Foo , Bar , ", toSet("Foo", "Bar")}, - {" Foo Bar ", toSet("Foo Bar")}, - {" , ", toSet()}, - {" ", toSet()}, - {"", toSet()}, - {"=?UTF-8?Q?Archived,Category_personal,test_=F0=9F=98=80=F0=9F=99=83?=", toSet("Archived", "Category personal", "test 😀🙃")}, - {"IMAP_NotJunk,Foo,Opened,bar,Unread", toSet("Foo", "bar")}, - } - for _, tc := range tests { - tc := tc - t.Run(fmt.Sprintf("%v", tc.value), func(t *testing.T) { - labels := getGmailLabelsFromValue(tc.value) - r.Equal(t, tc.wantLabels, labels) - }) - } -} - -func toSet(items ...string) stringSet { - set := map[string]bool{} - for _, item := range items { - set[item] = true - } - return set -} diff --git a/internal/transfer/provider_mbox_source.go b/internal/transfer/provider_mbox_source.go deleted file mode 100644 index 68ada1d1..00000000 --- a/internal/transfer/provider_mbox_source.go +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - - "github.com/emersion/go-mbox" - "github.com/pkg/errors" -) - -// TransferTo exports messages based on rules to channel. -func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) { - log.Info("Started transfer from MBOX to channel") - defer log.Info("Finished transfer from MBOX to channel") - - filePathsPerFolder, err := p.getFilePathsPerFolder() - if err != nil { - progress.fatal(err) - return - } - - if len(filePathsPerFolder) == 0 { - return - } - - for folderName, filePaths := range filePathsPerFolder { - log.WithField("folder", folderName).Debug("Estimating folder counts") - for _, filePath := range filePaths { - if progress.shouldStop() { - break - } - p.updateCount(progress, filePath) - } - } - progress.countsFinal() - - for folderName, filePaths := range filePathsPerFolder { - log.WithField("folder", folderName).Debug("Processing folder") - for _, filePath := range filePaths { - if progress.shouldStop() { - break - } - p.transferTo(rules, progress, ch, folderName, filePath) - } - } -} - -func (p *MBOXProvider) getFilePathsPerFolder() (map[string][]string, error) { - filePaths, err := getAllPathsWithSuffix(p.root, ".mbox") - if err != nil { - return nil, err - } - - filePathsMap := map[string][]string{} - for _, filePath := range filePaths { - fileName := filepath.Base(filePath) - filePath, err := p.handleAppleMailMBOXStructure(filePath) - // Skip unsupported MBOX structures. It was already filtered out in configuration step. - if err != nil { - continue - } - - folder := strings.TrimSuffix(fileName, ".mbox") - filePathsMap[folder] = append(filePathsMap[folder], filePath) - } - return filePathsMap, nil -} - -func (p *MBOXProvider) updateCount(progress *Progress, filePath string) { - mboxReader := p.openMbox(progress, filePath) - if mboxReader == nil { - return - } - - count := 0 - for { - _, err := mboxReader.NextMessage() - if err == io.EOF { - break - } else if err != nil { - progress.fatal(err) - break - } - count++ - } - progress.updateCount(filePath, uint(count)) -} - -func (p *MBOXProvider) transferTo(rules transferRules, progress *Progress, ch chan<- Message, folderName, filePath string) { - mboxReader := p.openMbox(progress, filePath) - if mboxReader == nil { - return - } - - index := 0 - for { - if progress.shouldStop() { - break - } - - index++ - id := fmt.Sprintf("%s:%d", filePath, index) - - msgReader, err := mboxReader.NextMessage() - if err == io.EOF { - break - } else if err != nil { - progress.fatal(err) - break - } - - msg, err := p.exportMessage(rules, folderName, id, msgReader) - - progress.addMessage(id, msg.sourceNames(), msg.targetNames()) - - if err == nil && len(msg.Targets) == 0 { - progress.messageSkipped(id) - continue - } - - progress.messageExported(id, msg.Body, err) - if err == nil { - ch <- msg - } - } -} - -func (p *MBOXProvider) exportMessage(rules transferRules, folderName, id string, msgReader io.Reader) (Message, error) { - body, err := ioutil.ReadAll(msgReader) - if err != nil { - return Message{}, errors.Wrap(err, "failed to read message") - } - - msgRules := p.getMessageRules(rules, folderName, id, body) - sources := p.getMessageSources(msgRules) - targets := p.getMessageTargets(msgRules, id, body) - return Message{ - ID: id, - Unread: false, - Body: body, - Sources: sources, - Targets: targets, - }, nil -} - -func (p *MBOXProvider) getMessageRules(rules transferRules, folderName, id string, body []byte) []*Rule { - msgRules := []*Rule{} - - folderRule, err := rules.getRuleBySourceMailboxName(folderName) - if err != nil { - log.WithField("msg", id).WithField("source", folderName).Debug("Message source doesn't have a rule") - } else if folderRule.Active { - msgRules = append(msgRules, folderRule) - } - - gmailLabels, err := getGmailLabelsFromMessage(body) - if err != nil { - log.WithError(err).Error("Failed to get gmail labels, ") - } else { - for label := range gmailLabels { - rule, err := rules.getRuleBySourceMailboxName(label) - if err != nil { - log.WithField("msg", id).WithField("source", label).Debug("Message source doesn't have a rule") - continue - } - if rule.Active { - msgRules = append(msgRules, rule) - } - } - } - - return msgRules -} - -func (p *MBOXProvider) getMessageSources(msgRules []*Rule) []Mailbox { - sources := []Mailbox{} - for _, rule := range msgRules { - sources = append(sources, rule.SourceMailbox) - } - return sources -} - -func (p *MBOXProvider) getMessageTargets(msgRules []*Rule, id string, body []byte) []Mailbox { - targets := []Mailbox{} - haveExclusiveMailbox := false - for _, rule := range msgRules { - // Read and check time in body only if the rule specifies it - // to not waste energy. - if rule.HasTimeLimit() { - msgTime, err := getMessageTime(body) - if err != nil { - log.WithError(err).Error("Failed to parse time, time check skipped") - } else if !rule.isTimeInRange(msgTime) { - log.WithField("msg", id).WithField("source", rule.SourceMailbox.Name).Debug("Message skipped due to time") - continue - } - } - for _, newTarget := range rule.TargetMailboxes { - // msgRules is sorted. The first rule is based on the folder name, - // followed by the order from X-Gmail-Labels. The rule based on - // the folder name should have priority for exclusive target. - if newTarget.IsExclusive && haveExclusiveMailbox { - continue - } - found := false - for _, target := range targets { - if target.Hash() == newTarget.Hash() { - found = true - break - } - } - if found { - continue - } - if newTarget.IsExclusive { - haveExclusiveMailbox = true - } - targets = append(targets, newTarget) - } - } - return targets -} - -func (p *MBOXProvider) openMbox(progress *Progress, mboxPath string) *mbox.Reader { - mboxPath = filepath.Join(p.root, mboxPath) - mboxFile, err := os.Open(mboxPath) //nolint[gosec] - if os.IsNotExist(err) { - return nil - } else if err != nil { - progress.fatal(err) - return nil - } - return mbox.NewReader(mboxFile) -} diff --git a/internal/transfer/provider_mbox_target.go b/internal/transfer/provider_mbox_target.go deleted file mode 100644 index 82f43018..00000000 --- a/internal/transfer/provider_mbox_target.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "os" - "path/filepath" - "strings" - "time" - - "github.com/emersion/go-mbox" - "github.com/hashicorp/go-multierror" -) - -// DefaultMailboxes returns the default mailboxes for default rules if no other is found. -func (p *MBOXProvider) DefaultMailboxes(sourceMailbox Mailbox) []Mailbox { - return []Mailbox{{ - Name: sourceMailbox.Name, - }} -} - -// CreateMailbox does nothing. Files are created dynamically during the import. -func (p *MBOXProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) { - return mailbox, nil -} - -// TransferFrom imports messages from channel. -func (p *MBOXProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) { - log.Info("Started transfer from channel to MBOX") - defer log.Info("Finished transfer from channel to MBOX") - - for msg := range ch { - if progress.shouldStop() { - break - } - - err := p.writeMessage(msg) - progress.messageImported(msg.ID, "", err) - } -} - -func (p *MBOXProvider) writeMessage(msg Message) error { - var multiErr error - for _, mailbox := range msg.Targets { - mboxName := sanitizeFileName(mailbox.Name) - if !strings.HasSuffix(mboxName, ".mbox") { - mboxName += ".mbox" - } - - mboxPath := filepath.Join(p.root, mboxName) - mboxFile, err := os.OpenFile(filepath.Clean(mboxPath), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - multiErr = multierror.Append(multiErr, err) - continue - } - - msgFrom := "" - msgTime := time.Now() - if header, err := getMessageHeader(msg.Body); err == nil { - if date, err := header.Date(); err == nil { - msgTime = date - } - if addresses, err := header.AddressList("from"); err == nil && len(addresses) > 0 { - msgFrom = addresses[0].Address - } - } - - mboxWriter := mbox.NewWriter(mboxFile) - messageWriter, err := mboxWriter.CreateMessage(msgFrom, msgTime) - if err != nil { - multiErr = multierror.Append(multiErr, err) - continue - } - - _, err = messageWriter.Write(msg.Body) - if err != nil { - multiErr = multierror.Append(multiErr, err) - continue - } - } - return multiErr -} diff --git a/internal/transfer/provider_mbox_test.go b/internal/transfer/provider_mbox_test.go deleted file mode 100644 index 83cabdad..00000000 --- a/internal/transfer/provider_mbox_test.go +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "fmt" - "io/ioutil" - "os" - "testing" - "time" - - r "github.com/stretchr/testify/require" -) - -func newTestMBOXProvider(path string) *MBOXProvider { - if path == "" { - path = "testdata/mbox" - } - return NewMBOXProvider(path) -} - -func TestMBOXProviderMailboxes(t *testing.T) { - tests := []struct { - provider *MBOXProvider - includeEmpty bool - wantMailboxes []Mailbox - }{ - {newTestMBOXProvider(""), true, []Mailbox{ - {Name: "All Mail"}, - {Name: "Foo"}, - {Name: "Bar"}, - {Name: "Inbox"}, - }}, - {newTestMBOXProvider(""), false, []Mailbox{ - {Name: "All Mail"}, - {Name: "Foo"}, - {Name: "Bar"}, - {Name: "Inbox"}, - }}, - {newTestMBOXProvider("testdata/mbox-applemail"), true, []Mailbox{ - {Name: "All Mail"}, - {Name: "Foo"}, - {Name: "Bar"}, - }}, - } - for _, tc := range tests { - tc := tc - t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) { - mailboxes, err := tc.provider.Mailboxes(tc.includeEmpty, false) - r.NoError(t, err) - r.ElementsMatch(t, tc.wantMailboxes, mailboxes) - }) - } -} - -func TestMBOXProviderTransferTo(t *testing.T) { - provider := newTestMBOXProvider("") - - rules, rulesClose := newTestRules(t) - defer rulesClose() - setupMBOXRules(rules) - - msgs := testTransferTo(t, rules, provider, []string{ - "All Mail.mbox:1", - "All Mail.mbox:2", - "Foo.mbox:1", - "Inbox.mbox:1", - }) - got := map[string][]string{} - for _, msg := range msgs { - got[msg.ID] = msg.targetNames() - } - r.Equal(t, map[string][]string{ - "All Mail.mbox:1": {"Archive", "Foo"}, // Bar is not in rules. - "All Mail.mbox:2": {"Archive", "Foo"}, - "Foo.mbox:1": {"Foo"}, - "Inbox.mbox:1": {"Inbox"}, - }, got) -} - -func TestMBOXProviderTransferToAppleMail(t *testing.T) { - provider := newTestMBOXProvider("testdata/mbox-applemail") - - rules, rulesClose := newTestRules(t) - defer rulesClose() - setupMBOXRules(rules) - - msgs := testTransferTo(t, rules, provider, []string{ - "All Mail.mbox/mbox:1", - "All Mail.mbox/mbox:2", - }) - got := map[string][]string{} - for _, msg := range msgs { - got[msg.ID] = msg.targetNames() - } - r.Equal(t, map[string][]string{ - "All Mail.mbox/mbox:1": {"Archive", "Foo"}, // Bar is not in rules. - "All Mail.mbox/mbox:2": {"Archive", "Foo"}, - }, got) -} - -func TestMBOXProviderTransferFrom(t *testing.T) { - dir, err := ioutil.TempDir("", "mbox") - r.NoError(t, err) - defer os.RemoveAll(dir) //nolint[errcheck] - - provider := newTestMBOXProvider(dir) - - rules, rulesClose := newTestRules(t) - defer rulesClose() - setupMBOXRules(rules) - - testTransferFrom(t, rules, provider, []Message{ - {ID: "Foo.mbox:1", Body: getTestMsgBody("msg"), Targets: []Mailbox{{Name: "Foo"}}}, - }) - - checkMBOXFileStructure(t, dir, []string{ - "Foo.mbox", - }) -} - -func TestMBOXProviderTransferFromTo(t *testing.T) { - dir, err := ioutil.TempDir("", "mbox") - r.NoError(t, err) - defer os.RemoveAll(dir) //nolint[errcheck] - - source := newTestMBOXProvider("") - target := newTestMBOXProvider(dir) - - rules, rulesClose := newTestRules(t) - defer rulesClose() - setupMBOXRules(rules) - - testTransferFromTo(t, rules, source, target, 5*time.Second) - - checkMBOXFileStructure(t, dir, []string{ - "Archive.mbox", - "Foo.mbox", - "Inbox.mbox", - }) -} - -func TestMBOXProviderGetMessageRules(t *testing.T) { - provider := newTestMBOXProvider("") - - body := []byte(`Subject: Test -X-Gmail-Labels: foo,bar - -`) - rules := transferRules{ - rules: map[string]*Rule{ - "1": {Active: true, SourceMailbox: Mailbox{Name: "folder"}}, - "2": {Active: false, SourceMailbox: Mailbox{Name: "foo"}}, - "3": {Active: true, SourceMailbox: Mailbox{Name: "bar"}}, - "4": {Active: false, SourceMailbox: Mailbox{Name: "baz"}}, - "5": {Active: true, SourceMailbox: Mailbox{Name: "other"}}, - }, - } - - gotRules := provider.getMessageRules(rules, "folder", "id", body) - r.Equal(t, 2, len(gotRules)) - r.Equal(t, "folder", gotRules[0].SourceMailbox.Name) - r.Equal(t, "bar", gotRules[1].SourceMailbox.Name) -} - -func TestMBOXProviderGetMessageTargetsReturnsOnlyOneFolder(t *testing.T) { - provider := newTestMBOXProvider("") - - folderA := Mailbox{Name: "Folder A", IsExclusive: true} - folderB := Mailbox{Name: "Folder B", IsExclusive: true} - labelA := Mailbox{Name: "Label A", IsExclusive: false} - labelB := Mailbox{Name: "Label B", IsExclusive: false} - labelC := Mailbox{Name: "Label C", IsExclusive: false} - - rule1 := &Rule{TargetMailboxes: []Mailbox{folderA, labelA, labelB}} - rule2 := &Rule{TargetMailboxes: []Mailbox{folderB, labelC}} - rule3 := &Rule{TargetMailboxes: []Mailbox{folderB}} - - tests := []struct { - rules []*Rule - wantMailboxes []Mailbox - }{ - {[]*Rule{}, []Mailbox{}}, - {[]*Rule{rule1}, []Mailbox{folderA, labelA, labelB}}, - {[]*Rule{rule1, rule2}, []Mailbox{folderA, labelA, labelB, labelC}}, - {[]*Rule{rule1, rule3}, []Mailbox{folderA, labelA, labelB}}, - {[]*Rule{rule3, rule1}, []Mailbox{folderB, labelA, labelB}}, - } - for _, tc := range tests { - tc := tc - t.Run(fmt.Sprintf("%v", tc.rules), func(t *testing.T) { - mailboxes := provider.getMessageTargets(tc.rules, "", []byte("")) - r.Equal(t, tc.wantMailboxes, mailboxes) - }) - } -} - -func setupMBOXRules(rules transferRules) { - _ = rules.setRule(Mailbox{Name: "All Mail"}, []Mailbox{{Name: "Archive"}}, 0, 0) - _ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0) - _ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0) -} - -func checkMBOXFileStructure(t *testing.T, root string, expectedFiles []string) { - files, err := getAllPathsWithSuffix(root, ".mbox") - r.NoError(t, err) - r.Equal(t, expectedFiles, files) -} diff --git a/internal/transfer/provider_pmapi.go b/internal/transfer/provider_pmapi.go deleted file mode 100644 index 8097a696..00000000 --- a/internal/transfer/provider_pmapi.go +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "context" - "sort" - - "github.com/ProtonMail/gopenpgp/v2/crypto" - "github.com/ProtonMail/proton-bridge/pkg/message" - "github.com/ProtonMail/proton-bridge/pkg/pmapi" - "github.com/pkg/errors" -) - -const ( - fetchWorkers = 20 // In how many workers to fetch message (group list on IMAP). - attachWorkers = 5 // In how many workers to fetch attachments (for one message). - buildWorkers = 20 // In how many workers to build messages. -) - -// PMAPIProvider implements import and export to/from ProtonMail server. -type PMAPIProvider struct { - client pmapi.Client - userID string - addressID string - keyRing *crypto.KeyRing - builder *message.Builder - - nextImportRequests map[string]*pmapi.ImportMsgReq // Key is msg transfer ID. - nextImportRequestsSize int - - timeIt *timeIt - - connection bool -} - -// NewPMAPIProvider returns new PMAPIProvider. -func NewPMAPIProvider(client pmapi.Client, userID, addressID string) (*PMAPIProvider, error) { - provider := &PMAPIProvider{ - client: client, - userID: userID, - addressID: addressID, - builder: message.NewBuilder(fetchWorkers, attachWorkers, buildWorkers), - - nextImportRequests: map[string]*pmapi.ImportMsgReq{}, - nextImportRequestsSize: 0, - - timeIt: newTimeIt("pmapi"), - } - - if addressID != "" { - keyRing, err := client.KeyRingForAddressID(addressID) - if err != nil { - return nil, errors.Wrap(err, "failed to get key ring") - } - provider.keyRing = keyRing - } - - return provider, nil -} - -// ID returns identifier of current setup of PMAPI provider. -// Identification is unique per user. -func (p *PMAPIProvider) ID() string { - return p.userID -} - -// Mailboxes returns all available labels in ProtonMail account. -func (p *PMAPIProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) { - labels, err := p.client.ListLabels(context.Background()) - if err != nil { - return nil, err - } - sortedLabels := byFoldersLabels(labels) - sort.Sort(sortedLabels) - - emptyLabelsMap := map[string]bool{} - if !includeEmpty { - messagesCounts, err := p.client.CountMessages(context.Background(), p.addressID) - if err != nil { - return nil, err - } - for _, messagesCount := range messagesCounts { - if messagesCount.Total == 0 { - emptyLabelsMap[messagesCount.LabelID] = true - } - } - } - - mailboxes := []Mailbox{} - for _, mailbox := range getSystemMailboxes(includeAllMail) { - if !includeEmpty && emptyLabelsMap[mailbox.ID] { - continue - } - - mailboxes = append(mailboxes, mailbox) - } - for _, label := range sortedLabels { - if !includeEmpty && emptyLabelsMap[label.ID] { - continue - } - - mailboxes = append(mailboxes, Mailbox{ - ID: label.ID, - Name: label.Name, - Color: label.Color, - IsExclusive: bool(label.Exclusive), - }) - } - return mailboxes, nil -} - -func getSystemMailboxes(includeAllMail bool) []Mailbox { - mailboxes := []Mailbox{ - {ID: pmapi.InboxLabel, Name: "Inbox", IsExclusive: true}, - {ID: pmapi.DraftLabel, Name: "Drafts", IsExclusive: true}, - {ID: pmapi.SentLabel, Name: "Sent", IsExclusive: true}, - {ID: pmapi.StarredLabel, Name: "Starred", IsExclusive: true}, - {ID: pmapi.ArchiveLabel, Name: "Archive", IsExclusive: true}, - {ID: pmapi.SpamLabel, Name: "Spam", IsExclusive: true}, - {ID: pmapi.TrashLabel, Name: "Trash", IsExclusive: true}, - } - - if includeAllMail { - mailboxes = append(mailboxes, Mailbox{ - ID: pmapi.AllMailLabel, - Name: "All Mail", - IsExclusive: true, - }) - } - - return mailboxes -} - -type byFoldersLabels []*pmapi.Label - -func (l byFoldersLabels) Len() int { - return len(l) -} - -func (l byFoldersLabels) Swap(i, j int) { - l[i], l[j] = l[j], l[i] -} - -// Less sorts first folders, then labels, by user order. -func (l byFoldersLabels) Less(i, j int) bool { - if l[i].Exclusive && !l[j].Exclusive { - return true - } - if !l[i].Exclusive && l[j].Exclusive { - return false - } - return l[i].Order < l[j].Order -} diff --git a/internal/transfer/provider_pmapi_source.go b/internal/transfer/provider_pmapi_source.go deleted file mode 100644 index a54e58f5..00000000 --- a/internal/transfer/provider_pmapi_source.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "context" - "errors" - "fmt" - "sync" - - "github.com/ProtonMail/proton-bridge/pkg/message" - "github.com/ProtonMail/proton-bridge/pkg/pmapi" - "github.com/sirupsen/logrus" -) - -const pmapiListPageSize = 150 - -// TransferTo exports messages based on rules to channel. -func (p *PMAPIProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) { - log.Info("Started transfer from PMAPI to channel") - defer log.Info("Finished transfer from PMAPI to channel") - - p.timeIt.clear() - defer p.timeIt.logResults() - - // TransferTo cannot end sooner than loadCounts goroutine because - // loadCounts writes to channel in progress which would be closed. - // That can happen for really small accounts. - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - p.loadCounts(rules, progress) - }() - - for rule := range rules.iterateActiveRules() { - p.transferTo(rule, progress, ch, rules.skipEncryptedMessages) - } - - wg.Wait() -} - -func (p *PMAPIProvider) loadCounts(rules transferRules, progress *Progress) { - for rule := range rules.iterateActiveRules() { - if progress.shouldStop() { - break - } - - rule := rule - progress.callWrap(func() error { - _, total, err := p.listMessages(&pmapi.MessagesFilter{ - AddressID: p.addressID, - LabelID: rule.SourceMailbox.ID, - Begin: rule.FromTime, - End: rule.ToTime, - Limit: 0, - }) - if err != nil { - log.WithError(err).Warning("Problem to load counts") - return err - } - progress.updateCount(rule.SourceMailbox.Name, uint(total)) - return nil - }) - } - progress.countsFinal() -} - -func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, skipEncryptedMessages bool) { - page := 0 - for { - if progress.shouldStop() { - break - } - - isLastPage := true - - progress.callWrap(func() error { - // Would be better to filter by Begin and BeginID to be sure - // in case user deletes messages during the process, no message - // is skipped (paging is off then), but API does not support - // filtering by both mentioned fields at the same time. - desc := false - pmapiMessages, total, err := p.listMessages(&pmapi.MessagesFilter{ - AddressID: p.addressID, - LabelID: rule.SourceMailbox.ID, - Begin: rule.FromTime, - End: rule.ToTime, - PageSize: pmapiListPageSize, - Page: page, - Sort: "ID", - Desc: &desc, - }) - if err != nil { - return err - } - log.WithFields(logrus.Fields{ - "label": rule.SourceMailbox.ID, - "page": page, - "total": total, - "count": len(pmapiMessages), - }).Debug("Listing messages") - - isLastPage = len(pmapiMessages) < pmapiListPageSize - - for _, pmapiMessage := range pmapiMessages { - if progress.shouldStop() { - break - } - - msgID := fmt.Sprintf("%s_%s", rule.SourceMailbox.ID, pmapiMessage.ID) - progress.addMessage(msgID, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames()) - msg, err := p.exportMessage(rule, progress, pmapiMessage.ID, msgID, skipEncryptedMessages) - progress.messageExported(msgID, msg.Body, err) - if err == nil { - ch <- msg - } - } - - page++ - - return nil - }) - - if isLastPage { - break - } - } -} - -func (p *PMAPIProvider) exportMessage(rule *Rule, progress *Progress, pmapiMsgID, msgID string, skipEncryptedMessages bool) (Message, error) { - var msg *pmapi.Message - - progress.callWrap(func() error { - var err error - msg, err = p.getMessage(pmapiMsgID) - return err - }) - - p.timeIt.start("build", msgID) - defer p.timeIt.stop("build", msgID) - - body, err := p.builder.NewJobWithOptions( - context.Background(), - p.client, - msg.ID, - message.JobOptions{IgnoreDecryptionErrors: !skipEncryptedMessages}, - ).GetResult() - if err != nil { - if errors.Is(err, message.ErrDecryptionFailed) && skipEncryptedMessages { - err = errors.New("skipping encrypted message") - } - - return Message{Body: []byte(msg.Body)}, err - } - - return Message{ - ID: msgID, - Unread: bool(msg.Unread), - Body: body, - Sources: []Mailbox{rule.SourceMailbox}, - Targets: rule.TargetMailboxes, - }, nil -} diff --git a/internal/transfer/provider_pmapi_target.go b/internal/transfer/provider_pmapi_target.go deleted file mode 100644 index 4b27530d..00000000 --- a/internal/transfer/provider_pmapi_target.go +++ /dev/null @@ -1,339 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "bytes" - "context" - "fmt" - "io" - "io/ioutil" - "net/mail" - "sync" - - pkgMsg "github.com/ProtonMail/proton-bridge/pkg/message" - "github.com/ProtonMail/proton-bridge/pkg/pmapi" - "github.com/pkg/errors" -) - -const ( - pmapiImportBatchMaxItems = 10 - pmapiImportBatchMaxSize = 25 * 1000 * 1000 // 25 MB - pmapiImportWorkers = 4 // To keep memory under 1 GB. -) - -// DefaultMailboxes returns the default mailboxes for default rules if no other is found. -func (p *PMAPIProvider) DefaultMailboxes(_ Mailbox) []Mailbox { - return []Mailbox{{ - ID: pmapi.ArchiveLabel, - Name: "Archive", - IsExclusive: true, - }} -} - -// CreateMailbox creates label in ProtonMail account. -func (p *PMAPIProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) { - if mailbox.ID != "" { - return Mailbox{}, errors.New("mailbox is already created") - } - - label, err := p.client.CreateLabel(context.Background(), &pmapi.Label{ - Name: mailbox.Name, - Color: mailbox.Color, - Exclusive: pmapi.Boolean(mailbox.IsExclusive), - Type: pmapi.LabelTypeMailbox, - }) - if err != nil { - return Mailbox{}, errors.Wrap(err, fmt.Sprintf("failed to create mailbox %s", mailbox.Name)) - } - mailbox.ID = label.ID - return mailbox, nil -} - -// TransferFrom imports messages from channel. -func (p *PMAPIProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) { - log.Info("Started transfer from channel to PMAPI") - defer log.Info("Finished transfer from channel to PMAPI") - - p.timeIt.clear() - defer p.timeIt.logResults() - - // Cache has to be cleared before each transfer to not contain - // old stuff from previous cancelled run. - p.nextImportRequests = map[string]*pmapi.ImportMsgReq{} - p.nextImportRequestsSize = 0 - - preparedImportRequestsCh := make(chan map[string]*pmapi.ImportMsgReq) - wg := p.startImportWorkers(progress, preparedImportRequestsCh) - - for msg := range ch { - if progress.shouldStop() { - break - } - - if p.isMessageDraft(msg) { - p.transferDraft(rules, progress, msg) - } else { - p.transferMessage(rules, progress, msg, preparedImportRequestsCh) - } - } - - if len(p.nextImportRequests) > 0 { - preparedImportRequestsCh <- p.nextImportRequests - } - close(preparedImportRequestsCh) - wg.Wait() -} - -func (p *PMAPIProvider) isMessageDraft(msg Message) bool { - for _, target := range msg.Targets { - if target.ID == pmapi.DraftLabel { - return true - } - } - return false -} - -func (p *PMAPIProvider) transferDraft(rules transferRules, progress *Progress, msg Message) { - importedID, err := p.importDraft(msg, rules.globalMailbox) - progress.messageImported(msg.ID, importedID, err) -} - -func (p *PMAPIProvider) importDraft(msg Message, globalMailbox *Mailbox) (string, error) { //nolint[funlen] - message, attachmentReaders, err := p.parseMessage(msg) - if err != nil { - return "", errors.Wrap(err, "failed to parse message") - } - - if message.Sender == nil { - mainAddress := p.client.Addresses().Main() - message.Sender = &mail.Address{ - Name: mainAddress.DisplayName, - Address: mainAddress.Email, - } - } - - // Trying to encrypt an encrypted draft will return an error; - // users are forbidden to import messages encrypted with foreign keys to drafts. - if message.IsEncrypted() { - return "", errors.New("refusing to import draft encrypted by foreign key") - } - - p.timeIt.start("encrypt", msg.ID) - err = message.Encrypt(p.keyRing, nil) - p.timeIt.stop("encrypt", msg.ID) - if err != nil { - return "", errors.Wrap(err, "failed to encrypt draft") - } - - if globalMailbox != nil { - message.LabelIDs = append(message.LabelIDs, globalMailbox.ID) - } - - attachments := message.Attachments - message.Attachments = nil - - draft, err := p.createDraft(msg.ID, message, "", pmapi.DraftActionReply) - if err != nil { - return "", errors.Wrap(err, "failed to create draft") - } - - for idx, attachment := range attachments { - attachment.MessageID = draft.ID - attachmentBody, _ := ioutil.ReadAll(attachmentReaders[idx]) - - r := bytes.NewReader(attachmentBody) - sigReader, err := attachment.DetachedSign(p.keyRing, r) - if err != nil { - return "", errors.Wrap(err, "failed to sign attachment") - } - - p.timeIt.start("encrypt", msg.ID) - r = bytes.NewReader(attachmentBody) - encReader, err := attachment.Encrypt(p.keyRing, r) - p.timeIt.stop("encrypt", msg.ID) - if err != nil { - return "", errors.Wrap(err, "failed to encrypt attachment") - } - - _, err = p.createAttachment(msg.ID, attachment, encReader, sigReader) - if err != nil { - return "", errors.Wrap(err, "failed to create attachment") - } - } - - return draft.ID, nil -} - -func (p *PMAPIProvider) transferMessage(rules transferRules, progress *Progress, msg Message, preparedImportRequestsCh chan map[string]*pmapi.ImportMsgReq) { - importMsgReq, err := p.generateImportMsgReq(rules, progress, msg) - if err != nil { - progress.messageImported(msg.ID, "", err) - return - } - if importMsgReq == nil || progress.shouldStop() { - return - } - - importMsgReqSize := len(importMsgReq.Message) - if p.nextImportRequestsSize+importMsgReqSize > pmapiImportBatchMaxSize || len(p.nextImportRequests) == pmapiImportBatchMaxItems { - preparedImportRequestsCh <- p.nextImportRequests - p.nextImportRequests = map[string]*pmapi.ImportMsgReq{} - p.nextImportRequestsSize = 0 - } - p.nextImportRequests[msg.ID] = importMsgReq - p.nextImportRequestsSize += importMsgReqSize -} - -func (p *PMAPIProvider) generateImportMsgReq(rules transferRules, progress *Progress, msg Message) (*pmapi.ImportMsgReq, error) { - message, attachmentReaders, err := p.parseMessage(msg) - if err != nil { - return nil, errors.Wrap(err, "failed to parse message") - } - - var body []byte - if message.IsEncrypted() { - if rules.skipEncryptedMessages { - progress.messageSkipped(msg.ID) - return nil, nil - } - body = msg.Body - } else { - p.timeIt.start("encrypt", msg.ID) - body, err = p.encryptMessage(message, attachmentReaders) - p.timeIt.stop("encrypt", msg.ID) - if err != nil { - return nil, errors.Wrap(err, "failed to encrypt message") - } - } - - labelIDs := []string{} - for _, target := range msg.Targets { - // Frontend should not set All Mail to Rules, but to be sure... - if target.ID != pmapi.AllMailLabel { - labelIDs = append(labelIDs, target.ID) - } - } - if rules.globalMailbox != nil { - labelIDs = append(labelIDs, rules.globalMailbox.ID) - } - - return &pmapi.ImportMsgReq{ - Metadata: &pmapi.ImportMetadata{ - AddressID: p.addressID, - Unread: pmapi.Boolean(msg.Unread), - Time: message.Time, - Flags: computeMessageFlags(message.Header), - LabelIDs: labelIDs, - }, - Message: body, - }, nil -} - -func (p *PMAPIProvider) parseMessage(msg Message) (m *pmapi.Message, r []io.Reader, err error) { - p.timeIt.start("parse", msg.ID) - defer p.timeIt.stop("parse", msg.ID) - message, _, _, attachmentReaders, err := pkgMsg.Parse(bytes.NewBuffer(msg.Body)) - return message, attachmentReaders, err -} - -func (p *PMAPIProvider) encryptMessage(msg *pmapi.Message, attachmentReaders []io.Reader) ([]byte, error) { - if msg.MIMEType == pmapi.ContentTypeMultipartEncrypted { - return []byte(msg.Body), nil - } - return pkgMsg.BuildEncrypted(msg, attachmentReaders, p.keyRing) -} - -func computeMessageFlags(header mail.Header) (flag int64) { - if header.Get("received") == "" { - return pmapi.FlagSent - } - return pmapi.FlagReceived -} - -func (p *PMAPIProvider) startImportWorkers(progress *Progress, preparedImportRequestsCh chan map[string]*pmapi.ImportMsgReq) *sync.WaitGroup { - var wg sync.WaitGroup - wg.Add(pmapiImportWorkers) - for i := 0; i < pmapiImportWorkers; i++ { - go func() { - for importRequests := range preparedImportRequestsCh { - p.importMessages(progress, importRequests) - } - wg.Done() - }() - } - return &wg -} - -func (p *PMAPIProvider) importMessages(progress *Progress, importRequests map[string]*pmapi.ImportMsgReq) { - if progress.shouldStop() { - return - } - - importMsgIDs := []string{} - importMsgRequests := pmapi.ImportMsgReqs{} - for msgID, req := range importRequests { - importMsgIDs = append(importMsgIDs, msgID) - importMsgRequests = append(importMsgRequests, req) - } - log.WithField("msgIDs", importMsgIDs).Trace("Importing messages") - results, err := p.importRequest(importMsgIDs[0], importMsgRequests) - - // In case the whole request failed, try to import every message one by one. - if err != nil || len(results) == 0 { - log.WithError(err).Warning("Importing messages failed, trying one by one") - for msgID, req := range importRequests { - importedID, err := p.importMessage(msgID, progress, req) - progress.messageImported(msgID, importedID, err) - } - return - } - - // In case request passed but some messages failed, try to import the failed ones alone. - for index, result := range results { - msgID := importMsgIDs[index] - if result.Error != nil { - log.WithError(result.Error).WithField("msg", msgID).Warning("Importing message failed, trying alone") - req := importMsgRequests[index] - importedID, err := p.importMessage(msgID, progress, req) - progress.messageImported(msgID, importedID, err) - } else { - progress.messageImported(msgID, result.MessageID, nil) - } - } -} - -func (p *PMAPIProvider) importMessage(msgSourceID string, progress *Progress, req *pmapi.ImportMsgReq) (importedID string, importedErr error) { - progress.callWrap(func() error { - results, err := p.importRequest(msgSourceID, pmapi.ImportMsgReqs{req}) - if err != nil { - return errors.Wrap(err, "failed to import messages") - } - if len(results) == 0 { - importedErr = errors.New("import ended with no result") - return nil // This should not happen, only when there is bug which means we should skip this one. - } - if results[0].Error != nil { - importedErr = errors.Wrap(results[0].Error, "failed to import message") - return nil //nolint[nilerr] Call passed but API refused this message, skip this one. - } - importedID = results[0].MessageID - return nil - }) - return importedID, importedErr -} diff --git a/internal/transfer/provider_pmapi_test.go b/internal/transfer/provider_pmapi_test.go deleted file mode 100644 index e15d3ad1..00000000 --- a/internal/transfer/provider_pmapi_test.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "bytes" - "context" - "fmt" - "testing" - "time" - - "github.com/ProtonMail/proton-bridge/pkg/pmapi" - gomock "github.com/golang/mock/gomock" - r "github.com/stretchr/testify/require" -) - -func TestPMAPIProviderMailboxes(t *testing.T) { - m := initMocks(t) - defer m.ctrl.Finish() - - setupPMAPIClientExpectationForExport(&m) - provider, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID") - r.NoError(t, err) - - tests := []struct { - includeEmpty bool - includeAllMail bool - wantMailboxes []Mailbox - }{ - {true, false, []Mailbox{ - {ID: "folder1", Name: "One", Color: "red", IsExclusive: true}, - {ID: "folder2", Name: "Two", Color: "orange", IsExclusive: true}, - {ID: "label2", Name: "Bar", Color: "green", IsExclusive: false}, - {ID: "label1", Name: "Foo", Color: "blue", IsExclusive: false}, - }}, - {false, true, []Mailbox{ - {ID: pmapi.AllMailLabel, Name: "All Mail", IsExclusive: true}, - {ID: "folder1", Name: "One", Color: "red", IsExclusive: true}, - {ID: "folder2", Name: "Two", Color: "orange", IsExclusive: true}, - {ID: "label1", Name: "Foo", Color: "blue", IsExclusive: false}, - }}, - } - for _, tc := range tests { - tc := tc - t.Run(fmt.Sprintf("%v-%v", tc.includeEmpty, tc.includeAllMail), func(t *testing.T) { - mailboxes, err := provider.Mailboxes(tc.includeEmpty, tc.includeAllMail) - r.NoError(t, err) - r.Equal(t, []Mailbox{ - {ID: pmapi.InboxLabel, Name: "Inbox", IsExclusive: true}, - {ID: pmapi.DraftLabel, Name: "Drafts", IsExclusive: true}, - {ID: pmapi.SentLabel, Name: "Sent", IsExclusive: true}, - {ID: pmapi.StarredLabel, Name: "Starred", IsExclusive: true}, - {ID: pmapi.ArchiveLabel, Name: "Archive", IsExclusive: true}, - {ID: pmapi.SpamLabel, Name: "Spam", IsExclusive: true}, - {ID: pmapi.TrashLabel, Name: "Trash", IsExclusive: true}, - }, mailboxes[:7]) - r.Equal(t, tc.wantMailboxes, mailboxes[7:]) - }) - } -} - -func TestPMAPIProviderTransferTo(t *testing.T) { - m := initMocks(t) - defer m.ctrl.Finish() - - setupPMAPIClientExpectationForExport(&m) - provider, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID") - r.NoError(t, err) - - rules, rulesClose := newTestRules(t) - defer rulesClose() - setupPMAPIRules(rules) - - testTransferTo(t, rules, provider, []string{ - "0_msg1", - "0_msg2", - }) -} - -func TestPMAPIProviderTransferFrom(t *testing.T) { - m := initMocks(t) - defer m.ctrl.Finish() - - setupPMAPIClientExpectationForImport(&m) - provider, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID") - r.NoError(t, err) - - rules, rulesClose := newTestRules(t) - defer rulesClose() - setupPMAPIRules(rules) - - testTransferFrom(t, rules, provider, []Message{ - {ID: "msg1", Body: getTestMsgBody("msg1"), Targets: []Mailbox{{ID: pmapi.InboxLabel}}}, - {ID: "msg2", Body: getTestMsgBody("msg2"), Targets: []Mailbox{{ID: pmapi.InboxLabel}}}, - }) -} - -func TestPMAPIProviderTransferFromDraft(t *testing.T) { - m := initMocks(t) - defer m.ctrl.Finish() - - setupPMAPIClientExpectationForImportDraft(&m) - provider, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID") - r.NoError(t, err) - - rules, rulesClose := newTestRules(t) - defer rulesClose() - setupPMAPIRules(rules) - - testTransferFrom(t, rules, provider, []Message{ - {ID: "draft1", Body: getTestMsgBody("draft1"), Targets: []Mailbox{{ID: pmapi.DraftLabel}}}, - }) -} - -func TestPMAPIProviderTransferFromTo(t *testing.T) { - m := initMocks(t) - defer m.ctrl.Finish() - - setupPMAPIClientExpectationForExport(&m) - setupPMAPIClientExpectationForImport(&m) - - source, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID") - r.NoError(t, err) - target, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID") - r.NoError(t, err) - - rules, rulesClose := newTestRules(t) - defer rulesClose() - setupPMAPIRules(rules) - - testTransferFromTo(t, rules, source, target, 5*time.Second) -} - -func setupPMAPIRules(rules transferRules) { - _ = rules.setRule(Mailbox{ID: pmapi.InboxLabel}, []Mailbox{{ID: pmapi.InboxLabel}}, 0, 0) -} - -func setupPMAPIClientExpectationForExport(m *mocks) { - m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes() - m.pmapiClient.EXPECT().ListLabels(gomock.Any()).Return([]*pmapi.Label{ - {ID: "label1", Name: "Foo", Color: "blue", Exclusive: false, Order: 2}, - {ID: "label2", Name: "Bar", Color: "green", Exclusive: false, Order: 1}, - {ID: "folder1", Name: "One", Color: "red", Exclusive: true, Order: 1}, - {ID: "folder2", Name: "Two", Color: "orange", Exclusive: true, Order: 2}, - }, nil).AnyTimes() - m.pmapiClient.EXPECT().CountMessages(gomock.Any(), gomock.Any()).Return([]*pmapi.MessagesCount{ - {LabelID: "label1", Total: 10}, - {LabelID: "label2", Total: 0}, - {LabelID: "folder1", Total: 20}, - }, nil).AnyTimes() - m.pmapiClient.EXPECT().ListMessages(gomock.Any(), gomock.Any()).Return([]*pmapi.Message{ - {ID: "msg1"}, - {ID: "msg2"}, - }, 2, nil).AnyTimes() - m.pmapiClient.EXPECT().GetMessage(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, msgID string) (*pmapi.Message, error) { - return &pmapi.Message{ - ID: msgID, - Body: string(getTestMsgBody(msgID)), - MIMEType: pmapi.ContentTypeMultipartMixed, - }, nil - }).AnyTimes() -} - -func setupPMAPIClientExpectationForImport(m *mocks) { - m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes() - m.pmapiClient.EXPECT().Import(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, requests pmapi.ImportMsgReqs) ([]*pmapi.ImportMsgRes, error) { - results := []*pmapi.ImportMsgRes{} - for _, request := range requests { - for _, msgID := range []string{"msg1", "msg2"} { - if bytes.Contains(request.Message, []byte(msgID)) { - results = append(results, &pmapi.ImportMsgRes{MessageID: msgID, Error: nil}) - } - } - } - return results, nil - }).AnyTimes() -} - -func setupPMAPIClientExpectationForImportDraft(m *mocks) { - m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes() - m.pmapiClient.EXPECT().CreateDraft(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, msg *pmapi.Message, parentID string, action int) (*pmapi.Message, error) { - r.Equal(m.t, msg.Subject, "draft1") - msg.ID = "draft1" - return msg, nil - }) -} diff --git a/internal/transfer/provider_pmapi_utils.go b/internal/transfer/provider_pmapi_utils.go deleted file mode 100644 index 2fefcacd..00000000 --- a/internal/transfer/provider_pmapi_utils.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "context" - "fmt" - "io" - "time" - - "github.com/ProtonMail/proton-bridge/pkg/pmapi" - "github.com/pkg/errors" -) - -const ( - pmapiRetries = 10 - pmapiReconnectTimeout = 30 * time.Minute - pmapiReconnectSleep = 10 * time.Second -) - -func (p *PMAPIProvider) SetConnectionUp() { - p.connection = true -} - -func (p *PMAPIProvider) SetConnectionDown() { - p.connection = false -} - -func (p *PMAPIProvider) ensureConnection(callback func() error) error { - var callErr error - for i := 1; i <= pmapiRetries; i++ { - callErr = callback() - if callErr == nil { - return nil - } - - log.WithField("attempt", i).WithError(callErr).Warning("API call failed, trying reconnect") - err := p.tryReconnect() - if err != nil { - return err - } - } - return errors.Wrap(callErr, "too many retries") -} - -func (p *PMAPIProvider) tryReconnect() error { - start := time.Now() - var previousErr error - for { - if time.Since(start) > pmapiReconnectTimeout { - return previousErr - } - - if !p.connection { - time.Sleep(pmapiReconnectSleep) - continue - } - - break - } - return nil -} - -func (p *PMAPIProvider) listMessages(filter *pmapi.MessagesFilter) (messages []*pmapi.Message, count int, err error) { - err = p.ensureConnection(func() error { - // Sort is used in the key so the filter is different for estimating and real fetching. - key := fmt.Sprintf("%s_%s_%d", filter.LabelID, filter.Sort, filter.Page) - p.timeIt.start("listing", key) - defer p.timeIt.stop("listing", key) - - messages, count, err = p.client.ListMessages(context.Background(), filter) - return err - }) - return -} - -func (p *PMAPIProvider) getMessage(msgID string) (message *pmapi.Message, err error) { - err = p.ensureConnection(func() error { - p.timeIt.start("download", msgID) - defer p.timeIt.stop("download", msgID) - - message, err = p.client.GetMessage(context.Background(), msgID) - return err - }) - return -} - -func (p *PMAPIProvider) importRequest(msgSourceID string, req pmapi.ImportMsgReqs) (res []*pmapi.ImportMsgRes, err error) { - err = p.ensureConnection(func() error { - p.timeIt.start("upload", msgSourceID) - defer p.timeIt.stop("upload", msgSourceID) - - res, err = p.client.Import(context.Background(), req) - return err - }) - return -} - -func (p *PMAPIProvider) createDraft(msgSourceID string, message *pmapi.Message, parent string, action int) (draft *pmapi.Message, err error) { - err = p.ensureConnection(func() error { - p.timeIt.start("upload", msgSourceID) - defer p.timeIt.stop("upload", msgSourceID) - - draft, err = p.client.CreateDraft(context.Background(), message, parent, action) - return err - }) - return -} - -func (p *PMAPIProvider) createAttachment(msgSourceID string, att *pmapi.Attachment, r io.Reader, sig io.Reader) (created *pmapi.Attachment, err error) { - err = p.ensureConnection(func() error { - // Use some attributes from attachment to have unique key for each call. - key := fmt.Sprintf("%s_%s_%d", msgSourceID, att.Name, att.Size) - p.timeIt.start("upload", key) - defer p.timeIt.stop("upload", key) - - created, err = p.client.CreateAttachment(context.Background(), att, r, sig) - return err - }) - return -} diff --git a/internal/transfer/provider_test.go b/internal/transfer/provider_test.go deleted file mode 100644 index 5dfbe18d..00000000 --- a/internal/transfer/provider_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "fmt" - "testing" - "time" - - a "github.com/stretchr/testify/assert" - r "github.com/stretchr/testify/require" -) - -func getTestMsgBody(subject string) []byte { - return []byte(fmt.Sprintf(`Subject: %s -From: Bridge Test -To: Bridge Test -Content-Type: multipart/mixed; boundary=c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a - ---c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a -Content-Disposition: inline -Content-Transfer-Encoding: 7bit -Content-Type: text/plain; charset=utf-8 - -hello - ---c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a-- -`, subject)) -} - -func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, expectedMessageIDs []string) []Message { - progress := newProgress(log, nil) - drainProgressUpdateChannel(&progress) - - ch := make(chan Message) - go func() { - provider.TransferTo(rules, &progress, ch) - close(ch) - }() - - msgs := []Message{} - gotMessageIDs := []string{} - for msg := range ch { - msgs = append(msgs, msg) - gotMessageIDs = append(gotMessageIDs, msg.ID) - } - r.ElementsMatch(t, expectedMessageIDs, gotMessageIDs) - - r.Empty(t, progress.GetFailedMessages()) - - return msgs -} - -func testTransferFrom(t *testing.T, rules transferRules, provider TargetProvider, messages []Message) { - progress := newProgress(log, nil) - drainProgressUpdateChannel(&progress) - - ch := make(chan Message) - go func() { - for _, message := range messages { - progress.addMessage(message.ID, []string{}, []string{}) - progress.messageExported(message.ID, []byte(""), nil) - ch <- message - } - close(ch) - }() - - go func() { - provider.TransferFrom(rules, &progress, ch) - progress.finish() - }() - - maxWait := time.Duration(len(messages)*2) * time.Second - a.Eventually(t, func() bool { - return progress.updateCh == nil - }, maxWait, 10*time.Millisecond, "Waiting for imported messages timed out") - - r.Empty(t, progress.GetFailedMessages()) -} - -func testTransferFromTo(t *testing.T, rules transferRules, source SourceProvider, target TargetProvider, maxWait time.Duration) { - progress := newProgress(log, nil) - drainProgressUpdateChannel(&progress) - - ch := make(chan Message) - go func() { - source.TransferTo(rules, &progress, ch) - close(ch) - }() - go func() { - target.TransferFrom(rules, &progress, ch) - progress.finish() - }() - - a.Eventually(t, func() bool { - return progress.updateCh == nil - }, maxWait, 10*time.Millisecond, "Waiting for export and import timed out") - - r.Empty(t, progress.GetFailedMessages()) -} diff --git a/internal/transfer/report.go b/internal/transfer/report.go deleted file mode 100644 index 61343225..00000000 --- a/internal/transfer/report.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/pkg/errors" -) - -// fileReport is struct which can write and read message details. -// File report includes private information. -type fileReport struct { - path string -} - -func openLastFileReport(reportsPath, importID string) (*fileReport, error) { //nolint[deadcode] - allLogFileNames, err := getFilePathsWithSuffix(reportsPath, ".log") - if err != nil { - return nil, err - } - - reportFileNames := []string{} - for _, fileName := range allLogFileNames { - if strings.HasPrefix(fileName, fmt.Sprintf("import_%s_", importID)) { - reportFileNames = append(reportFileNames, fileName) - } - } - if len(reportFileNames) == 0 { - return nil, errors.New("no report found") - } - - sort.Strings(reportFileNames) - reportFileName := reportFileNames[len(reportFileNames)-1] - path := filepath.Join(reportsPath, reportFileName) - return &fileReport{ - path: path, - }, nil -} - -func newFileReport(reportsPath, importID string) *fileReport { - fileName := fmt.Sprintf("import_%s_%d.log", importID, time.Now().Unix()) - path := filepath.Join(reportsPath, fileName) - - return &fileReport{ - path: path, - } -} - -func (r *fileReport) writeMessageStatus(messageStatus *MessageStatus) { - f, err := os.OpenFile(r.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) - if err != nil { - log.WithError(err).Error("Failed to open report file") - } - defer f.Close() //nolint[errcheck] - - messageReport := newMessageReportFromMessageStatus(messageStatus, true) - data, err := json.Marshal(messageReport) - if err != nil { - log.WithError(err).Error("Failed to marshall message details") - } - data = append(data, '\n') - - if _, err = f.Write(data); err != nil { - log.WithError(err).Error("Failed to write to report file") - } -} - -// bugReport is struct which can create report for bug reporting. -// Bug report does NOT include private information. -type bugReport struct { - data bytes.Buffer -} - -func (r *bugReport) writeMessageStatus(messageStatus *MessageStatus) { - messageReport := newMessageReportFromMessageStatus(messageStatus, false) - data, err := json.Marshal(messageReport) - if err != nil { - log.WithError(err).Error("Failed to marshall message details") - } - _, _ = r.data.Write(data) - _, _ = r.data.Write([]byte("\n")) -} - -func (r *bugReport) getData() []byte { - return r.data.Bytes() -} - -// messageReport is struct which holds data used by `fileReport` and `bugReport`. -type messageReport struct { - EventTime int64 - SourceID string - TargetID string - BodyHash string - SourceMailboxes []string - TargetMailboxes []string - Error string - - // Private information for user. - Subject string - From string - Time string -} - -func newMessageReportFromMessageStatus(messageStatus *MessageStatus, includePrivateInfo bool) messageReport { - md := messageReport{ - EventTime: messageStatus.eventTime.Unix(), - SourceID: messageStatus.SourceID, - TargetID: messageStatus.targetID, - BodyHash: messageStatus.bodyHash, - SourceMailboxes: messageStatus.sourceNames, - TargetMailboxes: messageStatus.targetNames, - Error: messageStatus.GetErrorMessage(), - } - - if includePrivateInfo { - md.Subject = messageStatus.Subject - md.From = messageStatus.From - md.Time = messageStatus.Time.Format(time.RFC1123Z) - } - - return md -} diff --git a/internal/transfer/rules.go b/internal/transfer/rules.go deleted file mode 100644 index cb98a258..00000000 --- a/internal/transfer/rules.go +++ /dev/null @@ -1,366 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/ProtonMail/proton-bridge/pkg/pmapi" - "github.com/pkg/errors" -) - -// transferRules maintains import rules, e.g. to which target mailbox should be -// source mailbox imported or what time spans. -type transferRules struct { - filePath string - - // rules is map with key as hash of source mailbox to its rule. - // Every source mailbox should have rule, at least disabled one. - rules map[string]*Rule - - // globalMailbox is applied to every message in the import phase. - // E.g., every message will be imported into this mailbox. - globalMailbox *Mailbox - - // globalFromTime and globalToTime is applied to every rule right - // before the transfer (propagateGlobalTime has to be called). - globalFromTime int64 - globalToTime int64 - - // skipEncryptedMessages determines whether message which cannot - // be decrypted should be imported/exported or skipped. - skipEncryptedMessages bool -} - -// loadRules loads rules from `rulesPath` based on `ruleID`. -func loadRules(rulesPath, ruleID string) transferRules { - fileName := fmt.Sprintf("rules_%s.json", ruleID) - filePath := filepath.Join(rulesPath, fileName) - - var rules map[string]*Rule - f, err := os.Open(filePath) //nolint[gosec] - if err != nil { - log.WithError(err).Debug("Problem to read rules") - } else { - defer f.Close() //nolint[errcheck] - if err := json.NewDecoder(f).Decode(&rules); err != nil { - log.WithError(err).Warn("Problem to umarshal rules") - } - } - if rules == nil { - rules = map[string]*Rule{} - } - - return transferRules{ - filePath: filePath, - rules: rules, - } -} - -func (r *transferRules) setSkipEncryptedMessages(skip bool) { - r.skipEncryptedMessages = skip -} - -func (r *transferRules) setGlobalMailbox(mailbox *Mailbox) { - r.globalMailbox = mailbox -} - -func (r *transferRules) setGlobalTimeLimit(fromTime, toTime int64) { - r.globalFromTime = fromTime - r.globalToTime = toTime -} - -func (r *transferRules) propagateGlobalTime() { - if r.globalFromTime == 0 && r.globalToTime == 0 { - return - } - for _, rule := range r.rules { - if !rule.HasTimeLimit() { - rule.FromTime = r.globalFromTime - rule.ToTime = r.globalToTime - } - } -} - -func (r *transferRules) getRuleBySourceMailboxName(name string) (*Rule, error) { - for _, rule := range r.rules { - if rule.SourceMailbox.Name == name { - return rule, nil - } - } - return nil, fmt.Errorf("no rule for mailbox %s", name) -} - -func (r *transferRules) iterateActiveRules() chan *Rule { - ch := make(chan *Rule) - go func() { - for _, rule := range r.rules { - if rule.Active { - ch <- rule - } - } - close(ch) - }() - return ch -} - -// setDefaultRules iterates `sourceMailboxes` and sets missing rules with -// matching mailboxes from `targetMailboxes`. In case no matching mailbox -// is found, `defaultCallback` with a source mailbox as a parameter is used. -func (r *transferRules) setDefaultRules(sourceMailboxes []Mailbox, targetMailboxes []Mailbox, defaultCallback func(Mailbox) []Mailbox) { - for _, sourceMailbox := range sourceMailboxes { - h := sourceMailbox.Hash() - if _, ok := r.rules[h]; ok { - continue - } - - targetMailboxes := sourceMailbox.findMatchingMailboxes(targetMailboxes) - - if !containsExclusive(targetMailboxes) { - targetMailboxes = append(targetMailboxes, defaultCallback(sourceMailbox)...) - } - - active := true - if len(targetMailboxes) == 0 { - active = false - } - - // For both import to or export from ProtonMail, spam and draft - // mailboxes are by default deactivated. - for _, mailbox := range append([]Mailbox{sourceMailbox}, targetMailboxes...) { - if mailbox.ID == pmapi.SpamLabel || mailbox.ID == pmapi.DraftLabel || mailbox.ID == pmapi.TrashLabel { - active = false - break - } - } - - r.rules[h] = &Rule{ - Active: active, - SourceMailbox: sourceMailbox, - TargetMailboxes: targetMailboxes, - } - } - - // There is no point showing rule which has no action (i.e., source mailbox - // is not available). - // A good reason to keep all rules and only deactivate them would be for - // multiple imports from different sources with the same or similar enough - // mailbox setup to reuse configuration. That is very minor feature which - // can be implemented in more reasonable way by allowing users to save and - // load configurations. - for key, rule := range r.rules { - found := false - for _, sourceMailbox := range sourceMailboxes { - if sourceMailbox.Name == rule.SourceMailbox.Name { - found = true - } - } - if !found { - delete(r.rules, key) - } - } - - r.save() -} - -// setRule sets messages from `sourceMailbox` between `fromData` and `toDate` -// (if used) to be imported to all `targetMailboxes`. -func (r *transferRules) setRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error { - numberOfExclusiveMailboxes := 0 - for _, mailbox := range targetMailboxes { - if mailbox.IsExclusive { - numberOfExclusiveMailboxes++ - } - } - if numberOfExclusiveMailboxes > 1 { - return errors.New("rule can have only one exclusive target mailbox") - } - - h := sourceMailbox.Hash() - r.rules[h] = &Rule{ - Active: true, - SourceMailbox: sourceMailbox, - TargetMailboxes: targetMailboxes, - FromTime: fromTime, - ToTime: toTime, - } - r.save() - return nil -} - -// unsetRule unsets messages from `sourceMailbox` to be exported. -func (r *transferRules) unsetRule(sourceMailbox Mailbox) { - h := sourceMailbox.Hash() - if rule, ok := r.rules[h]; ok { - rule.Active = false - } else { - r.rules[h] = &Rule{ - Active: false, - SourceMailbox: sourceMailbox, - } - } - r.save() -} - -// getRule returns rule for `sourceMailbox` or nil if it does not exist. -func (r *transferRules) getRule(sourceMailbox Mailbox) *Rule { - h := sourceMailbox.Hash() - return r.rules[h] -} - -// getSortedRules returns all set rules in order by `byRuleOrder`. -func (r *transferRules) getSortedRules() []*Rule { - rules := []*Rule{} - for _, rule := range r.rules { - rules = append(rules, rule) - } - sort.Sort(byRuleOrder(rules)) - return rules -} - -// reset wipes our all rules. -func (r *transferRules) reset() { - r.rules = map[string]*Rule{} - r.save() -} - -// save saves rules to file. -func (r *transferRules) save() { - f, err := os.Create(r.filePath) - if err != nil { - log.WithError(err).Warn("Problem to write rules") - return - } - defer f.Close() //nolint[errcheck] - - if err := json.NewEncoder(f).Encode(r.rules); err != nil { - log.WithError(err).Warn("Problem to marshal rules") - } -} - -// Rule is data holder of rule for one source mailbox used by `transferRules`. -type Rule struct { - Active bool `json:"active"` - SourceMailbox Mailbox `json:"source"` - TargetMailboxes []Mailbox `json:"targets"` - FromTime int64 `json:"from"` - ToTime int64 `json:"to"` -} - -// String returns textual representation for log purposes. -func (r *Rule) String() string { - return fmt.Sprintf( - "%s -> %s (%d - %d)", - r.SourceMailbox.Name, - strings.Join(r.TargetMailboxNames(), ", "), - r.FromTime, - r.ToTime, - ) -} - -func (r *Rule) isTimeInRange(t int64) bool { - if !r.HasTimeLimit() { - return true - } - return r.FromTime <= t && t <= r.ToTime -} - -// HasTimeLimit returns whether rule defines time limit. -func (r *Rule) HasTimeLimit() bool { - return r.FromTime != 0 || r.ToTime != 0 -} - -// FromDate returns time struct based on `FromTime`. -func (r *Rule) FromDate() time.Time { - return time.Unix(r.FromTime, 0) -} - -// ToDate returns time struct based on `ToTime`. -func (r *Rule) ToDate() time.Time { - return time.Unix(r.ToTime, 0) -} - -// TargetMailboxNames returns array of target mailbox names. -func (r *Rule) TargetMailboxNames() (names []string) { - for _, mailbox := range r.TargetMailboxes { - names = append(names, mailbox.Name) - } - return -} - -// byRuleOrder implements sort.Interface. Sort order: -// * System folders first (as defined in getSystemMailboxes). -// * Custom folders by name. -// * Custom labels by name. -type byRuleOrder []*Rule - -func (a byRuleOrder) Len() int { - return len(a) -} - -func (a byRuleOrder) Swap(i, j int) { - a[i], a[j] = a[j], a[i] -} - -func (a byRuleOrder) Less(i, j int) bool { - if a[i].SourceMailbox.IsExclusive && !a[j].SourceMailbox.IsExclusive { - return true - } - if !a[i].SourceMailbox.IsExclusive && a[j].SourceMailbox.IsExclusive { - return false - } - - iSystemIndex := -1 - jSystemIndex := -1 - for index, systemFolders := range getSystemMailboxes(true) { - if a[i].SourceMailbox.Name == systemFolders.Name { - iSystemIndex = index - } - if a[j].SourceMailbox.Name == systemFolders.Name { - jSystemIndex = index - } - } - if iSystemIndex != -1 && jSystemIndex == -1 { - return true - } - if iSystemIndex == -1 && jSystemIndex != -1 { - return false - } - if iSystemIndex != -1 && jSystemIndex != -1 { - return iSystemIndex < jSystemIndex - } - - return a[i].SourceMailbox.Name < a[j].SourceMailbox.Name -} - -// containsExclusive returns true if there is at least one exclusive mailbox. -func containsExclusive(mailboxes []Mailbox) bool { - for _, m := range mailboxes { - if m.IsExclusive { - return true - } - } - - return false -} diff --git a/internal/transfer/rules_test.go b/internal/transfer/rules_test.go deleted file mode 100644 index 29317a19..00000000 --- a/internal/transfer/rules_test.go +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "fmt" - "io/ioutil" - "os" - "testing" - - "github.com/ProtonMail/proton-bridge/pkg/pmapi" - r "github.com/stretchr/testify/require" -) - -func newTestRules(t *testing.T) (transferRules, func()) { - path, err := ioutil.TempDir("", "rules") - r.NoError(t, err) - - ruleID := "rule" - rules := loadRules(path, ruleID) - return rules, func() { - _ = os.RemoveAll(path) - } -} - -func TestLoadRules(t *testing.T) { - path, err := ioutil.TempDir("", "rules") - r.NoError(t, err) - defer os.RemoveAll(path) //nolint[errcheck] - - ruleID := "rule" - rules := loadRules(path, ruleID) - - mailboxA := Mailbox{ID: "1", Name: "One", Color: "orange", IsExclusive: true} - mailboxB := Mailbox{ID: "2", Name: "Two", Color: "", IsExclusive: true} - mailboxC := Mailbox{ID: "3", Name: "Three", Color: "", IsExclusive: false} - - r.NoError(t, rules.setRule(mailboxA, []Mailbox{mailboxB, mailboxC}, 0, 0)) - r.NoError(t, rules.setRule(mailboxB, []Mailbox{mailboxB}, 10, 20)) - r.NoError(t, rules.setRule(mailboxC, []Mailbox{}, 0, 30)) - - rules2 := loadRules(path, ruleID) - r.Equal(t, map[string]*Rule{ - mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB, mailboxC}, FromTime: 0, ToTime: 0}, - mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 10, ToTime: 20}, - mailboxC.Hash(): {Active: true, SourceMailbox: mailboxC, TargetMailboxes: []Mailbox{}, FromTime: 0, ToTime: 30}, - }, rules2.rules) - - rules2.unsetRule(mailboxA) - rules2.unsetRule(mailboxC) - - rules3 := loadRules(path, ruleID) - r.Equal(t, map[string]*Rule{ - mailboxA.Hash(): {Active: false, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB, mailboxC}, FromTime: 0, ToTime: 0}, - mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 10, ToTime: 20}, - mailboxC.Hash(): {Active: false, SourceMailbox: mailboxC, TargetMailboxes: []Mailbox{}, FromTime: 0, ToTime: 30}, - }, rules3.rules) -} - -func TestSetGlobalTimeLimit(t *testing.T) { - path, err := ioutil.TempDir("", "rules") - r.NoError(t, err) - defer os.RemoveAll(path) //nolint[errcheck] - - rules := loadRules(path, "rule") - - mailboxA := Mailbox{Name: "One"} - mailboxB := Mailbox{Name: "Two"} - - r.NoError(t, rules.setRule(mailboxA, []Mailbox{}, 10, 20)) - r.NoError(t, rules.setRule(mailboxB, []Mailbox{}, 0, 0)) - - rules.setGlobalTimeLimit(30, 40) - rules.propagateGlobalTime() - - r.Equal(t, map[string]*Rule{ - mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{}, FromTime: 10, ToTime: 20}, - mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{}, FromTime: 30, ToTime: 40}, - }, rules.rules) -} - -func TestSetDefaultRules(t *testing.T) { - path, err := ioutil.TempDir("", "rules") - r.NoError(t, err) - defer os.RemoveAll(path) //nolint[errcheck] - - rules := loadRules(path, "rule") - - mailbox1 := Mailbox{Name: "One"} // Set manually, default will not override it. - mailbox2 := Mailbox{Name: "Two"} // Matched by `targetMailboxes`. - mailbox3 := Mailbox{Name: "Three"} // Matched by `defaultCallback`, not included in `targetMailboxes`. - mailbox4 := Mailbox{Name: "Four"} // Matched by nothing, will not be active. - mailbox5 := Mailbox{Name: "Spam", ID: pmapi.SpamLabel} // Spam is inactive by default (ID found in source). - mailbox6a := Mailbox{Name: "Draft"} // Draft is inactive by default (ID found in target, mailbox6b). - mailbox6b := Mailbox{Name: "Draft", ID: pmapi.DraftLabel} - - sourceMailboxes := []Mailbox{mailbox1, mailbox2, mailbox3, mailbox4, mailbox5, mailbox6a} - targetMailboxes := []Mailbox{mailbox1, mailbox2, mailbox6b} - - r.NoError(t, rules.setRule(mailbox1, []Mailbox{mailbox3}, 0, 0)) - - defaultCallback := func(mailbox Mailbox) []Mailbox { - if mailbox.Name == "Three" { - return []Mailbox{mailbox3} - } - return []Mailbox{} - } - - rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback) - - r.Equal(t, map[string]*Rule{ - mailbox1.Hash(): {Active: true, SourceMailbox: mailbox1, TargetMailboxes: []Mailbox{mailbox3}}, - mailbox2.Hash(): {Active: true, SourceMailbox: mailbox2, TargetMailboxes: []Mailbox{mailbox2}}, - mailbox3.Hash(): {Active: true, SourceMailbox: mailbox3, TargetMailboxes: []Mailbox{mailbox3}}, - mailbox4.Hash(): {Active: false, SourceMailbox: mailbox4, TargetMailboxes: []Mailbox{}}, - mailbox5.Hash(): {Active: false, SourceMailbox: mailbox5, TargetMailboxes: []Mailbox{}}, - mailbox6a.Hash(): {Active: false, SourceMailbox: mailbox6a, TargetMailboxes: []Mailbox{mailbox6b}}, - }, rules.rules) -} - -func TestSetDefaultRulesDeactivateMissing(t *testing.T) { - path, err := ioutil.TempDir("", "rules") - r.NoError(t, err) - defer os.RemoveAll(path) //nolint[errcheck] - - rules := loadRules(path, "rule") - - mailboxA := Mailbox{ID: "1", Name: "One", Color: "", IsExclusive: true} - mailboxB := Mailbox{ID: "2", Name: "Two", Color: "", IsExclusive: true} - - r.NoError(t, rules.setRule(mailboxA, []Mailbox{mailboxB}, 0, 0)) - r.NoError(t, rules.setRule(mailboxB, []Mailbox{mailboxB}, 0, 0)) - - sourceMailboxes := []Mailbox{mailboxA} - targetMailboxes := []Mailbox{mailboxA, mailboxB} - defaultCallback := func(mailbox Mailbox) (mailboxes []Mailbox) { - return - } - rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback) - - r.Equal(t, map[string]*Rule{ - mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 0, ToTime: 0}, - }, rules.rules) -} - -func TestIsTimeInRange(t *testing.T) { - tests := []struct { - rule Rule - time int64 - want bool - }{ - {generateTimeRule(0, 0), 0, true}, - {generateTimeRule(0, 0), 10, true}, - {generateTimeRule(0, 15), 10, true}, - {generateTimeRule(5, 15), 10, true}, - {generateTimeRule(0, 5), 10, false}, - {generateTimeRule(5, 7), 10, false}, - {generateTimeRule(15, 30), 10, false}, - {generateTimeRule(15, 0), 10, false}, - } - for _, tc := range tests { - tc := tc - t.Run(fmt.Sprintf("%v / %d", tc.rule, tc.time), func(t *testing.T) { - got := tc.rule.isTimeInRange(tc.time) - r.Equal(t, tc.want, got) - }) - } -} - -func TestHasTimeLimit(t *testing.T) { - tests := []struct { - rule Rule - want bool - }{ - {generateTimeRule(0, 0), false}, - {generateTimeRule(0, 1), true}, - {generateTimeRule(1, 2), true}, - {generateTimeRule(1, 0), true}, - } - for _, tc := range tests { - tc := tc - t.Run(fmt.Sprintf("%v", tc.rule), func(t *testing.T) { - r.Equal(t, tc.want, tc.rule.HasTimeLimit()) - }) - } -} - -func generateTimeRule(from, to int64) Rule { - return Rule{ - SourceMailbox: Mailbox{}, - TargetMailboxes: []Mailbox{}, - FromTime: from, - ToTime: to, - } -} - -func TestOrderRules(t *testing.T) { - wantMailboxOrder := []Mailbox{ - {Name: "Inbox", IsExclusive: true}, - {Name: "Drafts", IsExclusive: true}, - {Name: "Sent", IsExclusive: true}, - {Name: "Starred", IsExclusive: true}, - {Name: "Archive", IsExclusive: true}, - {Name: "Spam", IsExclusive: true}, - {Name: "All Mail", IsExclusive: true}, - {Name: "Folder A", IsExclusive: true}, - {Name: "Folder B", IsExclusive: true}, - {Name: "Folder C", IsExclusive: true}, - {Name: "Label A", IsExclusive: false}, - {Name: "Label B", IsExclusive: false}, - {Name: "Label C", IsExclusive: false}, - } - wantMailboxNames := []string{} - - rules := map[string]*Rule{} - for _, mailbox := range wantMailboxOrder { - wantMailboxNames = append(wantMailboxNames, mailbox.Name) - rules[mailbox.Hash()] = &Rule{ - SourceMailbox: mailbox, - } - } - transferRules := transferRules{ - rules: rules, - } - - gotMailboxNames := []string{} - for _, rule := range transferRules.getSortedRules() { - gotMailboxNames = append(gotMailboxNames, rule.SourceMailbox.Name) - } - - r.Equal(t, wantMailboxNames, gotMailboxNames) -} diff --git a/internal/transfer/testdata/eml/Foo/msg.eml b/internal/transfer/testdata/eml/Foo/msg.eml deleted file mode 100644 index 3b342260..00000000 --- a/internal/transfer/testdata/eml/Foo/msg.eml +++ /dev/null @@ -1,4 +0,0 @@ -From: Bridge Test -To: Bridge Test - -hello diff --git a/internal/transfer/testdata/eml/Inbox/msg.eml b/internal/transfer/testdata/eml/Inbox/msg.eml deleted file mode 100644 index 3b342260..00000000 --- a/internal/transfer/testdata/eml/Inbox/msg.eml +++ /dev/null @@ -1,4 +0,0 @@ -From: Bridge Test -To: Bridge Test - -hello diff --git a/internal/transfer/testdata/emlmbox/Foo/msg.eml b/internal/transfer/testdata/emlmbox/Foo/msg.eml deleted file mode 100644 index 3b342260..00000000 --- a/internal/transfer/testdata/emlmbox/Foo/msg.eml +++ /dev/null @@ -1,4 +0,0 @@ -From: Bridge Test -To: Bridge Test - -hello diff --git a/internal/transfer/testdata/emlmbox/Inbox.mbox b/internal/transfer/testdata/emlmbox/Inbox.mbox deleted file mode 100644 index 25ad1c5b..00000000 --- a/internal/transfer/testdata/emlmbox/Inbox.mbox +++ /dev/null @@ -1,5 +0,0 @@ -From - Mon May 4 16:40:31 2020 -From: Bridge Test -To: Bridge Test - -hello diff --git a/internal/transfer/testdata/keyring_userKey b/internal/transfer/testdata/keyring_userKey deleted file mode 100644 index 976d2be2..00000000 --- a/internal/transfer/testdata/keyring_userKey +++ /dev/null @@ -1,62 +0,0 @@ ------BEGIN PGP PRIVATE KEY BLOCK----- -Version: OpenPGP.js v4.4.5 -Comment: testpassphrase - -xcLYBFzGzhEBCADBxfqTFMqfQzT77A5tuuhPFwPq8dfC2evs8u1OvTqFbztY -5FOuSxzduyeDqQ1Fx6dKEOKgcYE8t1Uh4VSS7z6bTdY8j9yrL81kCVB46sE1 -OzStzyx/5l7OdH/pM4F+aKslnLvqlw0UeJr+UNizVtOCEUaNfVjPK3cc1ocx -v+36K4RnnyfEtjUW9gDZbhgaF02G5ILHmWmbgM7I+77gCd2wI0EdY9s/JZQ+ -VmkMFqoMdY9PyBchoOIPUkkGQi1SaF4IEzMaAUSbnCYkHHY/SbfDTcR46VGq -cXlkB1rq5xskaUQ9r+giCC/K4pc7bBkI1lQ7ADVuWvdrWnWapK0FO6CfABEB -AAEAB/0YPhPJ0phA/EWviN+16bmGVOZNaVapjt2zMMybWmrtEQv3OeWgO3nP -4cohRi/zaCBCphcm+dxbLhftW7AFi/9PVcR09436MB+oTCQFugpUWw+4TmA5 -BidxTpDxf4X2vH3rquQLBufWL6U7JlPeKAGL1xZ2aCq0DIeOk5D+xTjZizV2 -GIyQRVCLWb+LfDmvvcp3Y94X60KXdBAMuS1ZMKcY3Sl8VAXNB4KQsC/kByzf -6FCB097XZRYV7lvJJQ7+6Wisb3yVi8sEQx2sFm5fAp+0qi3a6zRTEp49r6Hr -gyWViH5zOOpA7DcNwx1Bwhi7GG0tak6EUnnKUNLfOupglcphBADmpXCgT4nc -uSBYTiZSVcB/ICCkTxVsHL1WcXtPK2Ikzussx2n9kb0rapvuC0YLipX9lUkQ -fyeC3jQJeCyN79AkDGkOfWaESueT2hM0Po+RwDgMibKn6yJ1zebz4Lc2J3C9 -oVFcAnql+9KyGsAPn03fyQzDnvhNnJvHJi4Hx8AWoQQA1xLoXeVBjRi0IjjU -E6Mqaq5RLEog4kXRp86VSSEGHBwyIYnDiM//gjseo/CXuVyHwL7UXitp8s1B -D1uE3APrhqUS66fD5pkF+z+RcSqiIv7I76NJ24Cdg38L6seGSjOHrq7/dEeG -K6WqfQUCEjta3yNSg7pXb2wn2WZqKIK+rz8EALZRuMXeql/FtO3Cjb0sv7oT -9dLP4cn1bskGRJ+Vok9lfCERbfXGccoAk3V+qSfpHgKxsebkRbUhf+trOGnw -tW+kBWo/5hYGQuN+A9JogSJViT+nuZyE+x1/rKswDFmlMSdf2GIDARWIV0gc -b1yOEwUmNBSthPcnFXvBr4BG3XTtNPTNLSJhcm9uMjEtM0BzYWRlbWJlLm9y -ZyIgPGFyb24yMS0zQHNhZGVtYmUub3JnPsLAdQQQAQgAHwUCXMbOEQYLCQcI -AwIEFQgKAgMWAgECGQECGwMCHgEACgkQZ/B4v2b2xB6XUgf/dHGRHimyMR78 -QYbEm2cuaEvOtq4a+J6Zv3P4VOWAbvkGWS9LDKSvVi60vq4oYOmF54HgPzur -nA4OtZDf0HKwQK45VZ7CYD693o70jkKPrAAJG3yTsbesfiS7RbFyGKzKJ7EL -nsUIJkfgm/SlKmXU/u8MOBO5Wg7/TcsS33sRWHl90j+9jbhqdl92R+vY/CwC -ieFkQA7/TDv1u+NAalH+Lpkd8AIuEcki+TAogZ7oi/SnofwnoB7BxRm+mIkp -ZZhIDSCaPOzLG8CSZ81d3HVHhqbf8dh0DFKFoUYyKdbOqIkNWWASf+c/ZEme -IWcekY8hqwf/raZ56tGM/bRwYPcotMfC1wRcxs4RAQgAsMb5/ELWmrfPy3ba -5qif+RXhGSbjitATNgHpoPUHrfTC7cn4JWHqehoXLAQpFAoKd+O/ZNpZozK9 -ilpqGUx05yMw06jNQEhYIbgIF4wzPpz02Lp6YeMwdF5LF+Rw83PHdHrA/wRV -/QjL04+kZnN+G5HmzMlhFY+oZSpL+Gp1bTXgtAVDkhCnMB5tP2VwULMGyJ+X -vRYxwTK2CrLjIVZv5n1VYY+caCowU6j/XFqvlCJj+G5oV+UhFOWffaMRXhOh -a64RrhqT1Np7wCLvLMP2wpys9xlMcLQJLqDNxqOTp504V7dm67ncC0fKUsT4 -m4oTktnxKPd6MU+4VYveaLCquwARAQABAAf4u9s7gpGErs1USxmDO9TlyGZK -aBlri8nMf3s+hOJCOo3cRaRHJBfdY6pu/baG6H6JTsWzeY4MHwr6N+dhVIEh -FPMa9EZAjagyc4GugxWGiMVTfU+2AEfdrdynhQKMgXSctnnNCdkRuX0nwqb3 -nlupm1hsz2ze4+Wg0BKSLS0FQdoUbITdJUR69OHr4dNJVHWYI0JSBx4SdhV3 -y9163dDvmc+lW9AEaD53vyZWfzCHZxsR/gI32VmT0z5gn1t8w9AOdXo2lA1H -bf7wh4/qCyujGu64ToZtiEny/GCyM6PofLtiZuJNLw3s/y+B2tKv22aTJ760 -+Gib1xB9WcWjKyrxBADoeCyq+nHGrl0CwOkmjanlFymgo7mnBOXuiFOvGrKk -M1meMU1TI4TEBWkVnDVMcSejgjAf/bX1dtouba1tMAMu7DlaV/0EwbSADRel -RSqEbIzIOys+y9TY/BMI/uCKNyEKHvu1KUXADb+CBpdBpCfMBWDANFlo9xLz -Ajcmu2dyawQAwquwC0VXQcvzfs+Hd5au5XvHdm1KidOiAdu6PH1SrOgenIN4 -lkEjHrJD9jmloO2/GVcxDBB2pmf0B4HEg7DuY9LXBrksP5eSbbRc5+UH1HUv -u82AqQnfNKTd/jae+lLwaOS++ohtwMkkD6W0LdWnHPjyyXg4Oi9zPID3asRu -3PED/3CYyjl6S8GTMY4FNH7Yxu9+NV2xpKE92Hf8K/hnYlmSSVKDCEeOJtLt -BkkcSqY6liCNSMmJdVyAF2GrR+zmDac7UQRssf57oOWtSsGozt0aqJXuspMT -6aB+P1UhZ8Ly9rWZNiJ0jwyfnQNOLCYDaqjFmiSpqrNnJ2Q1Xge3+k80P9DC -wF8EGAEIAAkFAlzGzhECGwwACgkQZ/B4v2b2xB5wlwgAjZA1zdv5irFjyWVo -4/itONtyO1NbdpyYpcct7vD0oV+a4wahQP0J3Kk1GhZ5tvAoZF/jakQQOM5o -GjUYpXAGnr09Mv9EiQ2pDwXc2yq0WfXnGxNrpzOqdtV+IqY9NYkl55Tme7x+ -WRvrkPSUeUsyEGvxwR1stdv8eg9jUmxdl8Io3PYoFJJlrM/6aXeC1r3KOj7q -XAnR0XHJ+QBSNKCWLlQv5hui9BKfcLiVKFK/dNhs82nRyhPr4sWFw6MTqdAK -4zkn7l0jmy6Evi1AiiGPiHPnxeNErnofOIEh4REQj00deZADHrixTLtx2FuR -uaSC3IcBmBsj1fNb4eYXElILjQ== -=fMOl ------END PGP PRIVATE KEY BLOCK----- \ No newline at end of file diff --git a/internal/transfer/testdata/mbox-applemail/All Mail.mbox/mbox b/internal/transfer/testdata/mbox-applemail/All Mail.mbox/mbox deleted file mode 100644 index 2e758e86..00000000 --- a/internal/transfer/testdata/mbox-applemail/All Mail.mbox/mbox +++ /dev/null @@ -1,16 +0,0 @@ -From - Mon May 4 16:40:31 2020 -From: Bridge Test -To: Bridge Test -Subject: Test 1 -X-Gmail-Labels: Foo,Bar - -hello - - -From - Mon May 4 16:40:31 2020 -From: Bridge Test -To: Bridge Test -Subject: Test 2 -X-Gmail-Labels: Foo - -hello diff --git a/internal/transfer/testdata/mbox-applemail/Inbox.mbox/.keep b/internal/transfer/testdata/mbox-applemail/Inbox.mbox/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/internal/transfer/testdata/mbox/All Mail.mbox b/internal/transfer/testdata/mbox/All Mail.mbox deleted file mode 100644 index 2e758e86..00000000 --- a/internal/transfer/testdata/mbox/All Mail.mbox +++ /dev/null @@ -1,16 +0,0 @@ -From - Mon May 4 16:40:31 2020 -From: Bridge Test -To: Bridge Test -Subject: Test 1 -X-Gmail-Labels: Foo,Bar - -hello - - -From - Mon May 4 16:40:31 2020 -From: Bridge Test -To: Bridge Test -Subject: Test 2 -X-Gmail-Labels: Foo - -hello diff --git a/internal/transfer/testdata/mbox/Foo.mbox b/internal/transfer/testdata/mbox/Foo.mbox deleted file mode 100644 index 25ad1c5b..00000000 --- a/internal/transfer/testdata/mbox/Foo.mbox +++ /dev/null @@ -1,5 +0,0 @@ -From - Mon May 4 16:40:31 2020 -From: Bridge Test -To: Bridge Test - -hello diff --git a/internal/transfer/testdata/mbox/Inbox.mbox b/internal/transfer/testdata/mbox/Inbox.mbox deleted file mode 100644 index 25ad1c5b..00000000 --- a/internal/transfer/testdata/mbox/Inbox.mbox +++ /dev/null @@ -1,5 +0,0 @@ -From - Mon May 4 16:40:31 2020 -From: Bridge Test -To: Bridge Test - -hello diff --git a/internal/transfer/timeit.go b/internal/transfer/timeit.go deleted file mode 100644 index 34b49c64..00000000 --- a/internal/transfer/timeit.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "sync" - "time" -) - -type timeIt struct { - lock sync.Locker - name string - groups map[string]int64 - ongoing map[string]time.Time -} - -func newTimeIt(name string) *timeIt { - return &timeIt{ - lock: &sync.Mutex{}, - name: name, - groups: map[string]int64{}, - ongoing: map[string]time.Time{}, - } -} - -func (t *timeIt) clear() { - t.lock.Lock() - defer t.lock.Unlock() - - t.groups = map[string]int64{} - t.ongoing = map[string]time.Time{} -} - -func (t *timeIt) start(group, id string) { - t.lock.Lock() - defer t.lock.Unlock() - - t.ongoing[group+"/"+id] = time.Now() -} - -func (t *timeIt) stop(group, id string) { - endTime := time.Now() - - t.lock.Lock() - defer t.lock.Unlock() - - startTime, ok := t.ongoing[group+"/"+id] - if !ok { - log.WithField("group", group).WithField("id", id).Error("Stop called before start") - return - } - delete(t.ongoing, group+"/"+id) - - diff := endTime.Sub(startTime).Milliseconds() - t.groups[group] += diff -} - -func (t *timeIt) logResults() { - t.lock.Lock() - defer t.lock.Unlock() - - // Print also ongoing to be sure that nothing was left out. - // Basically ongoing should be empty. - log.WithField("name", t.name).WithField("result", t.groups).WithField("ongoing", t.ongoing).Debug("Time measurement") -} diff --git a/internal/transfer/transfer.go b/internal/transfer/transfer.go deleted file mode 100644 index a33e9b3b..00000000 --- a/internal/transfer/transfer.go +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -// Package transfer provides tools to export messages from one provider and -// import them to another provider. Provider can be EML, MBOX, IMAP or PMAPI. -package transfer - -import ( - "crypto/sha256" - "fmt" - - "github.com/sirupsen/logrus" -) - -var log = logrus.WithField("pkg", "transfer") //nolint[gochecknoglobals] - -// Transfer is facade on top of import rules, progress manager and source -// and target providers. This is the main object which should be used. -type Transfer struct { - panicHandler PanicHandler - metrics MetricsManager - id string - logDir string - rules transferRules - source SourceProvider - target TargetProvider - rulesCache []*Rule - sourceMboxCache []Mailbox - targetMboxCache []Mailbox -} - -// New creates Transfer for specific source and target. Usage: -// -// source := transfer.NewEMLProvider(...) -// target := transfer.NewPMAPIProvider(...) -// transfer.New(source, target, ...) -func New(panicHandler PanicHandler, metrics MetricsManager, logDir, rulesDir string, source SourceProvider, target TargetProvider) (*Transfer, error) { - transferID := fmt.Sprintf("%x", sha256.Sum256([]byte(source.ID()+"-"+target.ID()))) - rules := loadRules(rulesDir, transferID) - transfer := &Transfer{ - panicHandler: panicHandler, - metrics: metrics, - id: transferID, - logDir: logDir, - rules: rules, - source: source, - target: target, - } - if err := transfer.setDefaultRules(); err != nil { - return nil, err - } - metrics.Load(len(transfer.sourceMboxCache)) - return transfer, nil -} - -// SetDefaultRules sets missing rules for source mailboxes with matching -// target mailboxes. In case no matching mailbox is found, `defaultCallback` -// with a source mailbox as a parameter is used. -func (t *Transfer) setDefaultRules() error { - sourceMailboxes, err := t.SourceMailboxes() - if err != nil { - return err - } - - targetMailboxes, err := t.TargetMailboxes() - if err != nil { - return err - } - - defaultCallback := func(sourceMailbox Mailbox) []Mailbox { - return t.target.DefaultMailboxes(sourceMailbox) - } - - t.rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback) - return nil -} - -// SetSkipEncryptedMessages sets whether message which cannot be decrypted -// should be imported/exported or skipped. -func (t *Transfer) SetSkipEncryptedMessages(skip bool) { - t.rules.setSkipEncryptedMessages(skip) -} - -// SetGlobalMailbox sets mailbox that is applied to every message in -// the import phase. -func (t *Transfer) SetGlobalMailbox(mailbox *Mailbox) { - t.rules.setGlobalMailbox(mailbox) -} - -// SetGlobalTimeLimit sets time limit that is applied to rules without any -// specified time limit. -func (t *Transfer) SetGlobalTimeLimit(fromTime, toTime int64) { - t.rules.setGlobalTimeLimit(fromTime, toTime) -} - -// SetRule sets sourceMailbox for transfer. -func (t *Transfer) SetRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error { - t.rulesCache = nil - return t.rules.setRule(sourceMailbox, targetMailboxes, fromTime, toTime) -} - -// UnsetRule unsets sourceMailbox from transfer. -func (t *Transfer) UnsetRule(sourceMailbox Mailbox) { - t.rulesCache = nil - t.rules.unsetRule(sourceMailbox) -} - -// ResetRules unsets all rules. -func (t *Transfer) ResetRules() { - t.rulesCache = nil - t.rules.reset() -} - -// GetRule returns rule for given mailbox. -func (t *Transfer) GetRule(sourceMailbox Mailbox) *Rule { - return t.rules.getRule(sourceMailbox) -} - -// GetRules returns all set transfer rules. -func (t *Transfer) GetRules() []*Rule { - if t.rulesCache == nil { - t.rulesCache = t.rules.getSortedRules() - } - return t.rulesCache -} - -// SourceMailboxes returns mailboxes available at source side. -func (t *Transfer) SourceMailboxes() (m []Mailbox, err error) { - if t.sourceMboxCache == nil { - t.sourceMboxCache, err = t.source.Mailboxes(false, true) - } - return t.sourceMboxCache, err -} - -// TargetMailboxes returns mailboxes available at target side. -func (t *Transfer) TargetMailboxes() (m []Mailbox, err error) { - if t.targetMboxCache == nil { - t.targetMboxCache, err = t.target.Mailboxes(true, false) - } - return t.targetMboxCache, err -} - -// CreateTargetMailbox creates mailbox in target provider. -func (t *Transfer) CreateTargetMailbox(mailbox Mailbox) (Mailbox, error) { - t.targetMboxCache = nil - - return t.target.CreateMailbox(mailbox) -} - -// ChangeTarget changes the target. It is safe to change target for export, -// must not be changed for import. Do not set after you started transfer. -func (t *Transfer) ChangeTarget(target TargetProvider) { - t.targetMboxCache = nil - - t.target = target -} - -// Start starts the transfer from source to target. -func (t *Transfer) Start() *Progress { - log.Debug("Transfer started") - t.rules.save() - t.rules.propagateGlobalTime() - - t.metrics.Start() - - log := log.WithField("id", t.id) - reportFile := newFileReport(t.logDir, t.id) - progress := newProgress(log, reportFile) - - // Small queue to prevent having idle source while target is blocked. - // E.g., when upload to PM is in progress, we can in meantime download - // the next batch from remote IMAP server. - ch := make(chan Message, 10) - - go func() { - defer t.panicHandler.HandlePanic() - - t.source.TransferTo(t.rules, &progress, ch) - close(ch) - }() - - go func() { - defer t.panicHandler.HandlePanic() - - t.target.TransferFrom(t.rules, &progress, ch) - progress.finish() - - if progress.isStopped { - if progress.fatalError != nil { - t.metrics.Fail() - } else { - t.metrics.Cancel() - } - } else { - t.metrics.Complete() - } - }() - - return &progress -} diff --git a/internal/transfer/transfer_test.go b/internal/transfer/transfer_test.go deleted file mode 100644 index 1e27e9c6..00000000 --- a/internal/transfer/transfer_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "io/ioutil" - "testing" - - "github.com/ProtonMail/gopenpgp/v2/crypto" - transfermocks "github.com/ProtonMail/proton-bridge/internal/transfer/mocks" - pmapimocks "github.com/ProtonMail/proton-bridge/pkg/pmapi/mocks" - gomock "github.com/golang/mock/gomock" -) - -type mocks struct { - t *testing.T - - ctrl *gomock.Controller - panicHandler *transfermocks.MockPanicHandler - imapClientProvider *transfermocks.MockIMAPClientProvider - pmapiClient *pmapimocks.MockClient - - keyring *crypto.KeyRing -} - -func initMocks(t *testing.T) mocks { - mockCtrl := gomock.NewController(t) - - m := mocks{ - t: t, - - ctrl: mockCtrl, - panicHandler: transfermocks.NewMockPanicHandler(mockCtrl), - imapClientProvider: transfermocks.NewMockIMAPClientProvider(mockCtrl), - pmapiClient: pmapimocks.NewMockClient(mockCtrl), - keyring: newTestKeyring(), - } - - return m -} - -func newTestKeyring() *crypto.KeyRing { - data, err := ioutil.ReadFile("testdata/keyring_userKey") - if err != nil { - panic(err) - } - key, err := crypto.NewKeyFromArmored(string(data)) - if err != nil { - panic(err) - } - userKey, err := crypto.NewKeyRing(key) - if err != nil { - panic(err) - } - return userKey -} diff --git a/internal/transfer/types.go b/internal/transfer/types.go deleted file mode 100644 index 6a0b80d6..00000000 --- a/internal/transfer/types.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -type PanicHandler interface { - HandlePanic() -} - -type MetricsManager interface { - Load(int) - Start() - Complete() - Cancel() - Fail() -} diff --git a/internal/transfer/utils.go b/internal/transfer/utils.go deleted file mode 100644 index a9381b42..00000000 --- a/internal/transfer/utils.go +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "bufio" - "bytes" - "io/ioutil" - "net/mail" - "net/textproto" - "path/filepath" - "runtime" - "sort" - "strings" - - "github.com/ProtonMail/go-rfc5322" - "github.com/pkg/errors" -) - -// getFolderNames collects all folder names under `root`. -// Folder names will be without a path. -func getFolderNames(root string) ([]string, error) { - return getFolderNamesWithFileSuffix(root, "") -} - -// getFolderNamesWithFileSuffix collects all folder names under `root`, which -// contains some file with a give `fileSuffix`. Names will be without a path. -func getFolderNamesWithFileSuffix(root, fileSuffix string) ([]string, error) { - folders := []string{} - - files, err := ioutil.ReadDir(root) - if err != nil { - return nil, err - } - - hasFileWithSuffix := fileSuffix == "" - for _, file := range files { - if file.IsDir() { - subfolders, err := getFolderNamesWithFileSuffix(filepath.Join(root, file.Name()), fileSuffix) - if err != nil { - return nil, err - } - for _, subfolder := range subfolders { - match := false - for _, folder := range folders { - if folder == subfolder { - match = true - break - } - } - if !match { - folders = append(folders, subfolder) - } - } - } else if fileSuffix == "" || strings.HasSuffix(file.Name(), fileSuffix) { - hasFileWithSuffix = true - } - } - - if hasFileWithSuffix { - folders = append(folders, filepath.Base(root)) - } - - sort.Strings(folders) - return folders, nil -} - -// getFilePathsWithSuffix collects all file names with `suffix` under `root`. -// File names will be with relative path based to `root`. -func getFilePathsWithSuffix(root, suffix string) ([]string, error) { - fileNames, err := getFilePathsWithSuffixInner("", root, suffix, false) - if err != nil { - return nil, err - } - sort.Strings(fileNames) - return fileNames, err -} - -// getAllPathsWithSuffix is the same as getFilePathsWithSuffix but includes -// also directories. -func getAllPathsWithSuffix(root, suffix string) ([]string, error) { - fileNames, err := getFilePathsWithSuffixInner("", root, suffix, true) - if err != nil { - return nil, err - } - sort.Strings(fileNames) - return fileNames, err -} - -func getFilePathsWithSuffixInner(prefix, root, suffix string, includeDir bool) ([]string, error) { - fileNames := []string{} - - files, err := ioutil.ReadDir(root) - if err != nil { - return nil, err - } - - for _, file := range files { - if !file.IsDir() { - if strings.HasSuffix(file.Name(), suffix) { - fileNames = append(fileNames, filepath.Join(prefix, file.Name())) - } - } else { - if includeDir && strings.HasSuffix(file.Name(), suffix) { - fileNames = append(fileNames, filepath.Join(prefix, file.Name())) - } - subfolderFileNames, err := getFilePathsWithSuffixInner( - filepath.Join(prefix, file.Name()), - filepath.Join(root, file.Name()), - suffix, - includeDir, - ) - if err != nil { - return nil, err - } - fileNames = append(fileNames, subfolderFileNames...) - } - } - - return fileNames, nil -} - -// getMessageTime returns time of the message specified in the message header. -func getMessageTime(body []byte) (int64, error) { - hdr, err := getMessageHeader(body) - if err != nil { - return 0, err - } - - t, err := rfc5322.ParseDateTime(hdr.Get("Date")) - if err != nil { - return 0, err - } - - if t.IsZero() { - return 0, nil - } - - return t.Unix(), nil -} - -// getMessageHeader returns headers of the message body. -func getMessageHeader(body []byte) (mail.Header, error) { - tpr := textproto.NewReader(bufio.NewReader(bytes.NewBuffer(body))) - header, err := tpr.ReadMIMEHeader() - if err != nil { - return nil, errors.Wrap(err, "failed to read headers") - } - return mail.Header(header), nil -} - -// sanitizeFileName replaces problematic special characters with underscore. -func sanitizeFileName(fileName string) string { - if len(fileName) == 0 { - return fileName - } - if runtime.GOOS != "windows" && (fileName[0] == '-' || fileName[0] == '.') { //nolint[goconst] - fileName = "_" + fileName[1:] - } - return strings.Map(func(r rune) rune { - switch r { - case '\\', '/', ':', '*', '?', '"', '<', '>', '|': - return '_' - case '[', ']', '(', ')', '{', '}', '^', '#', '%', '&', '!', '@', '+', '=', '\'', '~': - if runtime.GOOS != "windows" { - return '_' - } - } - return r - }, fileName) -} diff --git a/internal/transfer/utils_test.go b/internal/transfer/utils_test.go deleted file mode 100644 index 3ec5fcb5..00000000 --- a/internal/transfer/utils_test.go +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package transfer - -import ( - "io/ioutil" - "os" - "path/filepath" - "runtime" - "testing" - - r "github.com/stretchr/testify/require" -) - -func TestGetFolderNames(t *testing.T) { - root, clean := createTestingFolderStructure(t) - defer clean() - - tests := []struct { - suffix string - wantNames []string - }{ - { - "", - []string{ - "bar", - "bar.mbox", - "baz", - filepath.Base(root), - "foo", - "qwerty", - "test", - }, - }, - { - ".eml", - []string{ - "bar", - "baz", - filepath.Base(root), - "foo", - }, - }, - { - ".txt", - []string{ - filepath.Base(root), - }, - }, - } - for _, tc := range tests { - tc := tc - t.Run(tc.suffix, func(t *testing.T) { - names, err := getFolderNamesWithFileSuffix(root, tc.suffix) - r.NoError(t, err) - r.Equal(t, tc.wantNames, names) - }) - } -} - -func TestGetFilePathsWithSuffix(t *testing.T) { - root, clean := createTestingFolderStructure(t) - defer clean() - - tests := []struct { - suffix string - wantPaths []string - }{ - { - ".eml", - []string{ - "foo/bar/baz/msg1.eml", - "foo/bar/baz/msg2.eml", - "foo/bar/baz/msg3.eml", - "foo/bar/msg4.eml", - "foo/bar/msg5.eml", - "foo/baz/msg6.eml", - "foo/msg7.eml", - "msg10.eml", - "test/foo/msg8.eml", - "test/foo/msg9.eml", - }, - }, - { - ".mbox", - []string{ - "bar.mbox", - "foo.mbox", - }, - }, - { - ".txt", - []string{ - "info.txt", - }, - }, - { - ".hello", - []string{}, - }, - } - for _, tc := range tests { - tc := tc - t.Run(tc.suffix, func(t *testing.T) { - paths, err := getAllPathsWithSuffix(root, tc.suffix) - r.NoError(t, err) - r.Equal(t, tc.wantPaths, paths) - }) - } -} - -func createTestingFolderStructure(t *testing.T) (string, func()) { - root, err := ioutil.TempDir("", "folderstructure") - r.NoError(t, err) - - for _, path := range []string{ - "foo/bar/baz", - "foo/baz", - "test/foo", - "qwerty", - "bar.mbox", - } { - err = os.MkdirAll(filepath.Join(root, path), os.ModePerm) - r.NoError(t, err) - } - - for _, path := range []string{ - "foo/bar/baz/msg1.eml", - "foo/bar/baz/msg2.eml", - "foo/bar/baz/msg3.eml", - "foo/bar/msg4.eml", - "foo/bar/msg5.eml", - "foo/baz/msg6.eml", - "foo/msg7.eml", - "test/foo/msg8.eml", - "test/foo/msg9.eml", - "msg10.eml", - "info.txt", - "foo.mbox", - "bar.mbox/mbox", // Apple Mail mbox export format. - } { - f, err := os.Create(filepath.Join(root, path)) - r.NoError(t, err) - err = f.Close() - r.NoError(t, err) - } - - return root, func() { - _ = os.RemoveAll(root) - } -} - -func TestGetMessageTime(t *testing.T) { - tests := []struct { - body string - wantTime int64 - wantErr string - }{ - {"", 0, "failed to read headers: EOF"}, - {"Subject: hello\n\n", 0, ""}, - {"Date: Thu, 23 Apr 2020 04:52:44 +0000\n\n", 1587617564, ""}, - } - for _, tc := range tests { - tc := tc - t.Run(tc.body, func(t *testing.T) { - time, err := getMessageTime([]byte(tc.body)) - if tc.wantErr == "" { - r.NoError(t, err) - } else { - r.EqualError(t, err, tc.wantErr) - } - r.Equal(t, tc.wantTime, time) - }) - } -} - -func TestGetMessageHeader(t *testing.T) { - body := `Subject: Hello -From: user@example.com - -Body -` - header, err := getMessageHeader([]byte(body)) - r.NoError(t, err) - r.Equal(t, header.Get("subject"), "Hello") - r.Equal(t, header.Get("from"), "user@example.com") -} - -func TestSanitizeFileName(t *testing.T) { - tests := map[string]string{ - "hello": "hello", - "a\\b/c:*?d\"<>|e": "a_b_c___d____e", - } - if runtime.GOOS == "darwin" || runtime.GOOS == "linux" { - tests[".hello"] = "_hello" - tests["-hello"] = "_hello" - } - if runtime.GOOS == "windows" { - tests["[hello]&@=~~"] = "_hello______" - } - - for path, wantPath := range tests { - path := path - wantPath := wantPath - t.Run(path, func(t *testing.T) { - gotPath := sanitizeFileName(path) - r.Equal(t, wantPath, gotPath) - }) - } -} diff --git a/internal/users/user.go b/internal/users/user.go index f1810e5d..8dd6921c 100644 --- a/internal/users/user.go +++ b/internal/users/user.go @@ -50,8 +50,6 @@ type User struct { creds *credentials.Credentials lock sync.RWMutex - - useOnlyActiveAddresses bool } // newUser creates a new user. @@ -62,7 +60,6 @@ func newUser( eventListener listener.Listener, credStorer CredentialsStorer, storeFactory StoreMaker, - useOnlyActiveAddresses bool, ) (*User, *credentials.Credentials, error) { log := log.WithField("user", userID) @@ -74,14 +71,13 @@ func newUser( } return &User{ - log: log, - panicHandler: panicHandler, - listener: eventListener, - credStorer: credStorer, - storeFactory: storeFactory, - userID: userID, - creds: creds, - useOnlyActiveAddresses: useOnlyActiveAddresses, + log: log, + panicHandler: panicHandler, + listener: eventListener, + credStorer: credStorer, + storeFactory: storeFactory, + userID: userID, + creds: creds, }, creds, nil } diff --git a/internal/users/user_credentials_test.go b/internal/users/user_credentials_test.go index a38b2734..accee921 100644 --- a/internal/users/user_credentials_test.go +++ b/internal/users/user_credentials_test.go @@ -172,7 +172,7 @@ func TestCheckBridgeLoginLoggedOut(t *testing.T) { m.eventListener.EXPECT().Emit(events.LogoutEvent, "user"), ) - user, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker, false) + user, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker) r.NoError(t, err) err = user.connect(m.pmapiClient, testCredentialsDisconnected) diff --git a/internal/users/user_new_test.go b/internal/users/user_new_test.go index 3b766180..f3810c06 100644 --- a/internal/users/user_new_test.go +++ b/internal/users/user_new_test.go @@ -33,7 +33,7 @@ func TestNewUserNoCredentialsStore(t *testing.T) { m.credentialsStore.EXPECT().Get("user").Return(nil, errors.New("fail")) - _, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker, false) + _, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker) r.Error(t, err) } @@ -71,7 +71,7 @@ func TestNewUser(t *testing.T) { } func checkNewUserHasCredentials(m mocks, wantErr string, wantCreds *credentials.Credentials) { - user, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker, false) + user, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker) r.NoError(m.t, err) defer cleanUpUserData(user) diff --git a/internal/users/user_test.go b/internal/users/user_test.go index 01d08de8..9ced4b94 100644 --- a/internal/users/user_test.go +++ b/internal/users/user_test.go @@ -27,7 +27,7 @@ func testNewUser(m mocks) *User { mockInitConnectedUser(m) mockEventLoopNoAction(m) - user, creds, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker, false) + user, creds, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker) r.NoError(m.t, err) err = user.connect(m.pmapiClient, creds) diff --git a/internal/users/users.go b/internal/users/users.go index 9e621d04..f7e211fd 100644 --- a/internal/users/users.go +++ b/internal/users/users.go @@ -58,12 +58,6 @@ type Users struct { // People are used to that and so we preserve that ordering here. users []*User - // useOnlyActiveAddresses determines whether credentials keeps only active - // addresses or all of them. Each usage has to be consisteng, e.g., once - // user is added, it saves address list to credentials and next time loads - // as is, without requesting server again. - useOnlyActiveAddresses bool - lock sync.RWMutex } @@ -74,19 +68,17 @@ func New( clientManager pmapi.Manager, credStorer CredentialsStorer, storeFactory StoreMaker, - useOnlyActiveAddresses bool, ) *Users { log.Trace("Creating new users") u := &Users{ - locations: locations, - panicHandler: panicHandler, - events: eventListener, - clientManager: clientManager, - credStorer: credStorer, - storeFactory: storeFactory, - useOnlyActiveAddresses: useOnlyActiveAddresses, - lock: sync.RWMutex{}, + locations: locations, + panicHandler: panicHandler, + events: eventListener, + clientManager: clientManager, + credStorer: credStorer, + storeFactory: storeFactory, + lock: sync.RWMutex{}, } go func() { @@ -135,7 +127,7 @@ func (u *Users) loadUsersFromCredentialsStore() error { for _, userID := range userIDs { l := log.WithField("user", userID) - user, creds, err := newUser(u.panicHandler, userID, u.events, u.credStorer, u.storeFactory, u.useOnlyActiveAddresses) + user, creds, err := newUser(u.panicHandler, userID, u.events, u.credStorer, u.storeFactory) if err != nil { l.WithError(err).Warn("Could not create user, skipping") continue @@ -256,19 +248,11 @@ func (u *Users) addNewUser(client pmapi.Client, apiUser *pmapi.User, auth *pmapi u.lock.Lock() defer u.lock.Unlock() - var emails []string - - if u.useOnlyActiveAddresses { - emails = client.Addresses().ActiveEmails() - } else { - emails = client.Addresses().AllEmails() - } - - if _, err := u.credStorer.Add(apiUser.ID, apiUser.Name, auth.UID, auth.RefreshToken, passphrase, emails); err != nil { + if _, err := u.credStorer.Add(apiUser.ID, apiUser.Name, auth.UID, auth.RefreshToken, passphrase, client.Addresses().ActiveEmails()); err != nil { return errors.Wrap(err, "failed to add user credentials to credentials store") } - user, creds, err := newUser(u.panicHandler, apiUser.ID, u.events, u.credStorer, u.storeFactory, u.useOnlyActiveAddresses) + user, creds, err := newUser(u.panicHandler, apiUser.ID, u.events, u.credStorer, u.storeFactory) if err != nil { return errors.Wrap(err, "failed to create new user") } diff --git a/internal/users/users_test.go b/internal/users/users_test.go index 962c8f7b..05077ec7 100644 --- a/internal/users/users_test.go +++ b/internal/users/users_test.go @@ -223,7 +223,7 @@ func testNewUsers(t *testing.T, m mocks) *Users { //nolint[unparam] m.eventListener.EXPECT().ProvideChannel(events.UpgradeApplicationEvent) m.eventListener.EXPECT().ProvideChannel(events.InternetOnEvent) - users := New(m.locator, m.PanicHandler, m.eventListener, m.clientManager, m.credentialsStore, m.storeMaker, true) + users := New(m.locator, m.PanicHandler, m.eventListener, m.clientManager, m.credentialsStore, m.storeMaker) waitForEvents() diff --git a/release-notes/ie_early.md b/release-notes/ie_early.md deleted file mode 100644 index f1acb692..00000000 --- a/release-notes/ie_early.md +++ /dev/null @@ -1,15 +0,0 @@ -## v1.3.0 -- 2021-02-02 - -### New - -- Introducing silent updates -- Improvements to message parsing - - -### Fixed - -- Setting up flags to avoid messages misplacement -- Remove dependency on go-apple-mobileconfig -- Change to how attachment size is processed to avoid potential errors -- Linux font issues - Fedora specific diff --git a/release-notes/ie_stable.md b/release-notes/ie_stable.md deleted file mode 100644 index 2528efa3..00000000 --- a/release-notes/ie_stable.md +++ /dev/null @@ -1,57 +0,0 @@ -## v1.3.3 -- 2021-05-17 - -### Fixed -- Fixed potential security vulnerability related to rpath -- Improved parsing of embedded messages - - -## v1.3.1 -- 2021-03-11 - -### New -- Reduce the number of import errors by supporting malformed undisclosed-recipient and better handling of overly long headers -- Improvements to how large attachments are processed -- New format of the release notes - -### Fixed -- Linux font issues - Fedora specific -- Fixed rare message misplacement -- Ensure removal of the startup entry during uninstallation -- Update errors - - -## v1.2.2 -- 2020-11-27 - -### New -- Improvements to the import from large mbox files with multiple labels -- Not allow to run multiple instances of the app or transfers at the same time -- Better handling and displaying of skipped messages -- Various enhancements of the import process related to parsing -- Cosmetic GUI changes -- Better error handling - -### Fixed -- Linux font issues - Fedora specific -- App response to the user pausing and canceling import or export -- Upgrade errors - - -## v1.1.2 -- 2020-09-23 - -### New -- Improving performance -- Speed up import by implementing parallel processing (parallel fetch, encrypt and upload of messages) -- Optimising the initial fetch of messages from external accounts -- Better message parsing -- Better handling of attachments and non-standard formatting -- Improved stability of the message parser -- Improved metrics -- Added persistent anonymous API cookies - -### Fixed -- Fixed issues causing failing of import -- Import from mbox files with long lines -- Improvements to import from Yahoo accounts diff --git a/test/Makefile b/test/Makefile index a876d723..993a83a1 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,10 +1,9 @@ -.PHONY: check-go check-godog install-godog test test-bridge test-ie test-live test-live-bridge test-live-ie test-stage test-debug test-live-debug bench +.PHONY: check-go check-godog install-godog test test-bridge test-live test-live-bridge test-stage test-debug test-live-debug bench export GO111MODULE=on export BRIDGE_VERSION:=1.8.12+integrationtests export VERBOSITY?=fatal export TEST_DATA=testdata -export TEST_APP?=bridge # Tests do not run in parallel. This will overrule user settings. MAKEFLAGS=-j1 @@ -16,23 +15,17 @@ check-godog: install-godog: check-go go get github.com/cucumber/godog/cmd/godog@v0.12.1 -test: test-bridge test-ie -test-bridge: FEATURES ?= features/bridge +test: test-bridge +test-bridge: FEATURES ?= features test-bridge: check-godog - TEST_APP=bridge TEST_ENV=fake TEST_ACCOUNTS=accounts/fake.json godog --tags="~@ignore" $(FEATURES) -test-ie: FEATURES ?= features/ie -test-ie: check-godog - TEST_APP=ie TEST_ENV=fake TEST_ACCOUNTS=accounts/fake.json godog --tags="~@ignore" $(FEATURES) + TEST_ENV=fake TEST_ACCOUNTS=accounts/fake.json godog --tags="~@ignore" $(FEATURES) # Doesn't work in parallel! # Provide TEST_ACCOUNTS with your accounts. test-live: test-live-bridge test-live-ie -test-live-bridge: FEATURES ?= features/bridge +test-live-bridge: FEATURES ?= features test-live-bridge: check-godog - TEST_APP=bridge TEST_ENV=live godog --tags="~@ignore && ~@ignore-live" $(FEATURES) -test-live-ie: FEATURES ?= features/ie -test-live-ie: check-godog - TEST_APP=ie TEST_ENV=live godog --tags="~@ignore && ~@ignore-live" $(FEATURES) + TEST_ENV=live godog --tags="~@ignore && ~@ignore-live" $(FEATURES) # Doesn't work in parallel! # Provide TEST_ACCOUNTS with your accounts. @@ -47,12 +40,6 @@ test-debug: test-live-debug: TEST_ENV=live dlv test -- $(FEATURES) -test-ie-debug: - TEST_APP=ie TEST_ENV=fake TEST_ACCOUNTS=accounts/fake.json dlv test -- $(FEATURES) - -test-live-ie-debug: - TEST_APP=ie TEST_ENV=live dlv test -- $(FEATURES) - # -run flag is not working anyway, but lets keep it there to note we really do not want to run tests. # To properly benchmark sync/fetch, we need everything empty. For that is better to start everything # again and safest way is to run only one loop per run. diff --git a/test/bdd_test.go b/test/bdd_test.go index b9f7cab9..469226dd 100644 --- a/test/bdd_test.go +++ b/test/bdd_test.go @@ -19,7 +19,6 @@ package tests import ( "context" - "os" testContext "github.com/ProtonMail/proton-bridge/test/context" "github.com/cucumber/godog" @@ -60,10 +59,6 @@ func ScenarioInitializer(s *godog.ScenarioContext) { StoreChecksFeatureContext(s) StoreSetupFeatureContext(s) - TransferActionsFeatureContext(s) - TransferChecksFeatureContext(s) - TransferSetupFeatureContext(s) - UsersActionsFeatureContext(s) UsersSetupFeatureContext(s) UsersChecksFeatureContext(s) @@ -72,9 +67,7 @@ func ScenarioInitializer(s *godog.ScenarioContext) { var ctx *testContext.TestContext //nolint[gochecknoglobals] func beforeScenario(scenarioCtx context.Context, _ *godog.Scenario) (context.Context, error) { - // NOTE(GODT-219) It would be possible to optimised the usage of godog with our context. - app := os.Getenv("TEST_APP") - ctx = testContext.New(app) + ctx = testContext.New() return scenarioCtx, nil } diff --git a/test/benchmarks/bench_test.go b/test/benchmarks/bench_test.go index c7212914..d61ba9c7 100644 --- a/test/benchmarks/bench_test.go +++ b/test/benchmarks/bench_test.go @@ -27,7 +27,7 @@ import ( ) func benchTestContext() (*context.TestContext, *mocks.IMAPClient) { - ctx := context.New("bridge") + ctx := context.New() username := "user" account := ctx.GetTestAccount(username) diff --git a/test/context/context.go b/test/context/context.go index 2008f4fd..9fcb1ee0 100644 --- a/test/context/context.go +++ b/test/context/context.go @@ -23,8 +23,6 @@ import ( "github.com/ProtonMail/proton-bridge/internal/bridge" "github.com/ProtonMail/proton-bridge/internal/config/useragent" - "github.com/ProtonMail/proton-bridge/internal/importexport" - "github.com/ProtonMail/proton-bridge/internal/transfer" "github.com/ProtonMail/proton-bridge/internal/users" "github.com/ProtonMail/proton-bridge/pkg/listener" "github.com/ProtonMail/proton-bridge/pkg/pmapi" @@ -55,11 +53,10 @@ type TestContext struct { clientManager pmapi.Manager // Core related variables. - bridge *bridge.Bridge - importExport *importexport.ImportExport - users *users.Users - credStore users.CredentialsStorer - lastError error + bridge *bridge.Bridge + users *users.Users + credStore users.CredentialsStorer + lastError error // IMAP related variables. imapAddr string @@ -75,13 +72,6 @@ type TestContext struct { smtpLastResponses map[string]*mocks.SMTPResponse smtpResponseLocker sync.Locker - // Transfer related variables. - transferLocalRootForImport string - transferLocalRootForExport string - transferRemoteIMAPServer *mocks.IMAPServer - transferProgress *transfer.Progress - transferSkipEncryptedMessages bool - // Store releated variables. bddMessageIDsToAPIIDs map[string]string @@ -93,9 +83,11 @@ type TestContext struct { } // New returns a new test TestContext. -func New(app string) *TestContext { +func New() *TestContext { + setLogrusVerbosityFromEnv() + listener := listener.New() - pmapiController, clientManager := newPMAPIController(app, listener) + pmapiController, clientManager := newPMAPIController(listener) ctx := &TestContext{ t: &bddT{}, @@ -121,15 +113,8 @@ func New(app string) *TestContext { // Ensure that the config is cleaned up after the test is over. ctx.addCleanupChecked(ctx.locations.Clear, "Cleaning bridge config data") - // Create bridge or import-export instance under test. - switch app { - case "bridge": - ctx.withBridgeInstance() - case "ie": - ctx.withImportExportInstance() - default: - panic("unknown app: " + app) - } + // Create bridge instance under test. + ctx.withBridgeInstance() return ctx } diff --git a/test/context/importexport.go b/test/context/importexport.go deleted file mode 100644 index ab738a27..00000000 --- a/test/context/importexport.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge.Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package context - -import ( - "github.com/ProtonMail/proton-bridge/internal/importexport" - "github.com/ProtonMail/proton-bridge/internal/users" - "github.com/ProtonMail/proton-bridge/pkg/listener" - "github.com/ProtonMail/proton-bridge/pkg/pmapi" -) - -// GetImportExport returns import-export instance. -func (ctx *TestContext) GetImportExport() *importexport.ImportExport { - return ctx.importExport -} - -// withImportExportInstance creates a import-export instance for use in the test. -// TestContext has this by default once called with env variable TEST_APP=ie. -func (ctx *TestContext) withImportExportInstance() { - ctx.importExport = newImportExportInstance(ctx.t, ctx.locations, ctx.cache, ctx.credStore, ctx.listener, ctx.clientManager) - ctx.users = ctx.importExport.Users -} - -// newImportExportInstance creates a new import-export instance configured to use the given config/credstore. -func newImportExportInstance( - t *bddT, - locations importexport.Locator, - cache importexport.Cacher, - credStore users.CredentialsStorer, - eventListener listener.Listener, - clientManager pmapi.Manager, -) *importexport.ImportExport { - panicHandler := &panicHandler{t: t} - return importexport.New(locations, cache, panicHandler, eventListener, clientManager, credStore) -} diff --git a/test/context/pmapi_controller.go b/test/context/pmapi_controller.go index de65a6d3..e32ca4c9 100644 --- a/test/context/pmapi_controller.go +++ b/test/context/pmapi_controller.go @@ -46,7 +46,7 @@ type PMAPIController interface { UnlockEvents() } -func newPMAPIController(app string, listener listener.Listener) (PMAPIController, pmapi.Manager) { +func newPMAPIController(listener listener.Listener) (PMAPIController, pmapi.Manager) { switch os.Getenv(EnvName) { case EnvFake: cntl, cm := fakeapi.NewController() @@ -54,7 +54,7 @@ func newPMAPIController(app string, listener listener.Listener) (PMAPIController return cntl, cm case EnvLive: - cntl, cm := liveapi.NewController(app) + cntl, cm := liveapi.NewController() addConnectionObserver(cm, listener) return cntl, cm diff --git a/test/context/transfer.go b/test/context/transfer.go deleted file mode 100644 index 7a8dd2c6..00000000 --- a/test/context/transfer.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge.Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package context - -import ( - "io/ioutil" - "math/rand" - "os" - "strconv" - - "github.com/ProtonMail/proton-bridge/internal/transfer" - "github.com/ProtonMail/proton-bridge/test/mocks" -) - -// SetTransferProgress sets transfer progress. -func (ctx *TestContext) SetTransferProgress(progress *transfer.Progress) { - ctx.transferProgress = progress -} - -// GetTransferProgress returns transfer progress. -func (ctx *TestContext) GetTransferProgress() *transfer.Progress { - return ctx.transferProgress -} - -// SetTransferSkipEncryptedMessages sets whether encrypted messages will be skipped. -func (ctx *TestContext) SetTransferSkipEncryptedMessages(value bool) { - ctx.transferSkipEncryptedMessages = value -} - -// GetTransferSkipEncryptedMessages gets whether encrypted messages will be skipped. -func (ctx *TestContext) GetTransferSkipEncryptedMessages() bool { - return ctx.transferSkipEncryptedMessages -} - -// GetTransferLocalRootForImport creates temporary root for importing -// if it not exists yet, and returns its path. -func (ctx *TestContext) GetTransferLocalRootForImport() string { - if ctx.transferLocalRootForImport != "" { - return ctx.transferLocalRootForImport - } - root := ctx.createLocalRoot() - ctx.transferLocalRootForImport = root - return root -} - -// GetTransferLocalRootForExport creates temporary root for exporting -// if it not exists yet, and returns its path. -func (ctx *TestContext) GetTransferLocalRootForExport() string { - if ctx.transferLocalRootForExport != "" { - return ctx.transferLocalRootForExport - } - root := ctx.createLocalRoot() - ctx.transferLocalRootForExport = root - return root -} - -func (ctx *TestContext) createLocalRoot() string { - root, err := ioutil.TempDir("", "transfer") - if err != nil { - panic("failed to create temp transfer root: " + err.Error()) - } - - ctx.addCleanupChecked(func() error { - return os.RemoveAll(root) - }, "Cleaning transfer data") - - return root -} - -// GetTransferRemoteIMAPServer creates mocked IMAP server if it not created yet, and returns it. -func (ctx *TestContext) GetTransferRemoteIMAPServer() *mocks.IMAPServer { - if ctx.transferRemoteIMAPServer != nil { - return ctx.transferRemoteIMAPServer - } - - port := 21300 + rand.Intn(100) //nolint[gosec] It is OK to use weaker rand generator here - ctx.transferRemoteIMAPServer = mocks.NewIMAPServer("user", "pass", "127.0.0.1", strconv.Itoa(port)) - - ctx.transferRemoteIMAPServer.Start() - ctx.addCleanupChecked(func() error { - ctx.transferRemoteIMAPServer.Stop() - return nil - }, "Cleaning transfer IMAP server") - - return ctx.transferRemoteIMAPServer -} diff --git a/test/features/ie/transfer/export_eml.feature b/test/features/ie/transfer/export_eml.feature deleted file mode 100644 index 50a7917c..00000000 --- a/test/features/ie/transfer/export_eml.feature +++ /dev/null @@ -1,43 +0,0 @@ -Feature: Export to EML files - Background: - Given there is connected user "user" - And there is "user" with mailbox "Folders/Foo" - And there are messages in mailbox "INBOX" for "user" - | from | to | subject | time | - | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 | - And there are messages in mailbox "Folders/Foo" for "user" - | from | to | subject | time | - | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 | - | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | - - Scenario: Export all - When user "user" exports to EML files - Then progress result is "OK" - # Every message is also in All Mail. - And transfer exported 8 messages - And transfer imported 8 messages - And transfer failed for 0 messages - And transfer exported messages - | folder | from | to | subject | time | - | Inbox | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 | - | Foo | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 | - | Foo | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - | Foo | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | - | All Mail | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 | - | All Mail | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 | - | All Mail | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - | All Mail | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | - - Scenario: Export only Foo with time limit - When user "user" exports to EML files with rules - | source | target | from | to | - | Foo | | 2020-01-01T12:10:00 | 2020-01-01T13:00:00 | - Then progress result is "OK" - And transfer exported 2 messages - And transfer imported 2 messages - And transfer failed for 0 messages - And transfer exported messages - | folder | from | to | subject | time | - | Foo | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - | Foo | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | diff --git a/test/features/ie/transfer/export_mbox.feature b/test/features/ie/transfer/export_mbox.feature deleted file mode 100644 index a0143eed..00000000 --- a/test/features/ie/transfer/export_mbox.feature +++ /dev/null @@ -1,43 +0,0 @@ -Feature: Export to MBOX files - Background: - Given there is connected user "user" - And there is "user" with mailbox "Folders/Foo" - And there are messages in mailbox "INBOX" for "user" - | from | to | subject | time | - | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 | - And there are messages in mailbox "Folders/Foo" for "user" - | from | to | subject | time | - | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 | - | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | - - Scenario: Export all - When user "user" exports to MBOX files - Then progress result is "OK" - # Every message is also in All Mail. - And transfer exported 8 messages - And transfer imported 8 messages - And transfer failed for 0 messages - And transfer exported messages - | folder | from | to | subject | time | - | Inbox | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 | - | Foo | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 | - | Foo | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - | Foo | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | - | All Mail | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 | - | All Mail | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 | - | All Mail | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - | All Mail | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | - - Scenario: Export only Foo with time limit - When user "user" exports to MBOX files with rules - | source | target | from | to | - | Foo | | 2020-01-01T12:10:00 | 2020-01-01T13:00:00 | - Then progress result is "OK" - And transfer exported 2 messages - And transfer imported 2 messages - And transfer failed for 0 messages - And transfer exported messages - | folder | from | to | subject | time | - | Foo | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - | Foo | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | diff --git a/test/features/ie/transfer/import_draft.feature b/test/features/ie/transfer/import_draft.feature deleted file mode 100644 index 83b11352..00000000 --- a/test/features/ie/transfer/import_draft.feature +++ /dev/null @@ -1,24 +0,0 @@ -Feature: Import from EML files - Background: - Given there is connected user "user" - - Scenario: Import draft without from fallbacks to primary address - Given there is EML file "Drafts/one.eml" - """ - Subject: no from yet - To: Internal Bridge - Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000 - - hello - - """ - When user "user" imports local files with rules - | source | target | - | Drafts | Drafts | - Then progress result is "OK" - And transfer exported 1 messages - And transfer imported 1 messages - And transfer failed for 0 messages - And API mailbox "Drafts" for "user" has messages - | from | to | subject | - | [userAddress] | test@protonmail.com | no from yet | diff --git a/test/features/ie/transfer/import_embedded.feature b/test/features/ie/transfer/import_embedded.feature deleted file mode 100644 index 59c53f09..00000000 --- a/test/features/ie/transfer/import_embedded.feature +++ /dev/null @@ -1,43 +0,0 @@ -Feature: Import embedded message - Background: - Given there is connected user "user" - And there is EML file "Inbox/hello.eml" - """ - From: Foo - To: Bridge Test - Subject: Embedded message - Content-Type: multipart/mixed; boundary="boundary" - Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000 - - This is a multi-part message in MIME format. - --boundary - Content-Type: text/plain; charset=utf-8 - Content-Transfer-Encoding: 7bit - - - --boundary - Content-Type: message/rfc822; name="embedded.eml" - Content-Transfer-Encoding: 7bit - Content-Disposition: attachment; filename="embedded.eml" - - From: Bar - To: Bridge Test - Subject: (No Subject) - Content-Type: text/plain; charset=utf-8 - Content-Transfer-Encoding: quoted-printable - - hello - - --boundary-- - - """ - - Scenario: Import it - When user "user" imports local files - Then progress result is "OK" - And transfer exported 1 messages - And transfer imported 1 messages - And transfer failed for 0 messages - And API mailbox "INBOX" for "user" has messages - | from | to | subject | - | foo@example.com | bridgetest@pm.test | Embedded message | diff --git a/test/features/ie/transfer/import_eml.feature b/test/features/ie/transfer/import_eml.feature deleted file mode 100644 index 87cbe313..00000000 --- a/test/features/ie/transfer/import_eml.feature +++ /dev/null @@ -1,61 +0,0 @@ -Feature: Import from EML files - Background: - Given there is connected user "user" - And there is "user" with mailbox "Folders/Foo" - And there is "user" with mailbox "Folders/Bar" - And there are EML files - | file | from | to | subject | time | - | Foo/one.eml | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 | - | Foo/two.eml | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - | Sub/Foo/three.eml | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | - And there is EML file "Inbox/hello.eml" - """ - Subject: hello - From: Bridge Test - To: Internal Bridge - Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000 - - hello - - """ - - Scenario: Import all - When user "user" imports local files - Then progress result is "OK" - And transfer exported 4 messages - And transfer imported 4 messages - And transfer failed for 0 messages - And API mailbox "INBOX" for "user" has messages - | from | to | subject | - | bridgetest@pm.test | test@protonmail.com | hello | - And API mailbox "Folders/Foo" for "user" has messages - | from | to | subject | - | foo@example.com | bridgetest@protonmail.com | one | - | bar@example.com | bridgetest@protonmail.com | two | - | bar@example.com | bridgetest@protonmail.com | three | - - Scenario: Import only Foo to Bar with time limit - When user "user" imports local files with rules - | source | target | from | to | - | Foo | Bar | 2020-01-01T12:10:00 | 2020-01-01T13:00:00 | - Then progress result is "OK" - And transfer exported 2 messages - And transfer imported 2 messages - And transfer failed for 0 messages - And API mailbox "Folders/Bar" for "user" has messages - | from | to | subject | - | bar@example.com | bridgetest@protonmail.com | two | - | bar@example.com | bridgetest@protonmail.com | three | - - Scenario: Import broken EML message - Given there is EML file "Broken/broken.eml" - """ - Content-type: multipart/mixed - """ - When user "user" imports local files with rules - | source | target | - | Broken | Foo | - Then progress result is "OK" - And transfer exported 1 messages - And transfer imported 0 messages - And transfer failed for 1 messages diff --git a/test/features/ie/transfer/import_encrypted.feature b/test/features/ie/transfer/import_encrypted.feature deleted file mode 100644 index 21d66595..00000000 --- a/test/features/ie/transfer/import_encrypted.feature +++ /dev/null @@ -1,188 +0,0 @@ -Feature: Import from EML files - Background: - Given there is connected user "user" - And there is EML file "Inbox/clear.eml" - """ - Subject: clear - From: Bridge Test - To: Internal Bridge - Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000 - - secret - """ - And there is EML file "Inbox/encrypted.eml" - """ - Subject: encrypted - From: Bridge Test - To: Internal Bridge - Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000 - - -----BEGIN PGP MESSAGE----- - - hQEMA7hGUUsYs0fEAQgA10NwJSNTLm3vpxVtoYBaA9AjFI5H4hKuV3/f2NHbWb2s - k5enK3tEIOYdFdrO+NLrV6weHq3Dgu4er3URTQ62tFwjSJyeXxmY0d9J+JdxibJg - wqZubuSHYzQHpFqJgoUUWEVEsv0Ao8yQo8BE9iybCKoZf6KojROUuE748oxlxJnV - m1XuaVIzgw4xN0GUA5sLLuWeL94b2dZe5SDDQE5POzDgueZ7faefX8U1pGErCRJ0 - sO6FSw3SF4NpvrxVESWgCmsG5pcuxE2JqB0UoHnNDcqsW8w1Q+GabAPo6UqHhgIg - 56MRCWeou2djHIIj9TMUIVFzSG/HvTYQWVS+i4Z7AdJJAXr53GgbZQznO80Qxwcb - FFdlwOXHuaXqhqCb338jlQWnbcnUsuJWxBAxkHrlP/nluFqPdIBOglC38kdYSBed - 3YwuEB9sXV/fcw== - =B05V - -----END PGP MESSAGE----- - """ - And there is EML file "Inbox/encrypted-mime.eml" - """ - Subject: encrypted mime - From: Bridge Test - To: Internal Bridge - Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000 - MIME-Version: 1.0 - Content-Type: multipart/encrypted; protocol="application/pgp-encrypted"; boundary="WLjzd46aUAiOcuNXjWTJItBZonI56MuAk" - - --WLjzd46aUAiOcuNXjWTJItBZonI56MuAk - Content-Type: application/pgp-encrypted - Content-Description: PGP/MIME version identification - - - --WLjzd46aUAiOcuNXjWTJItBZonI56MuAk - Content-Type: application/octet-stream; name="encrypted.asc" - Content-Description: OpenPGP encrypted message - Content-Disposition: inline; filename="encrypted.asc" - - -----BEGIN PGP MESSAGE----- - - hQEMA1ppSfinU0f4AQf9HDkojTV3SspsnhaB0HAsKIrUd+AAdSm49ndnJyjYb210 - GFIDE/TqcXmoeOcaJIRWaEOZzdcnixplJHjwp5dvDyCaYQSqYxUQ5Z/JfKbtsDyV - HbQzAh833SBCFlNNTnmF/Onu7yRNje1k8U36bY1VUX1QlerT9HDm2QTMRheuPDUR - H9OvGkuBXRpWRSPyXlPONPQOZTbUxvkuMGgDY0N2wt6kKQsrtduQNC157EJOErq0 - Zlhu9CnAyezDupMkSoikR1uyxo7GhyXNxi70Ol3tN7E2fnzeBCjUgmliYTABOGSH - nuPpTNk3/YoLEHXK18E/qR3vJTTl6AFIbOcfRCqpQIUBDAOjbxn1yC74AQEH/0kB - CiNDwPepRxwzv3EZT7V0YPuTCD18m9BZ4W5lVEvMNP7HJnCILJT8QJhLQ+AVBUuw - jhJqxAahssOGQ5BVxnWj4qwM+WzBOplH9Zt9bKTie8IdAJsl5GysL19jc4fjnvsK - weBQiR3Y+lEGEBrCajVrUkrXRHyA0fmel8aPfhiHxbh+jRtY8BWdBeX3gIfjwVKf - mMuTmHQ6ERv9CGpIy7mxRF67EIaVRhQzjNNnRlCIqgZHOpS72SKc6DtyCiR+ECjq - UAKNwOjTNZEzAjczyIB5Hkkw1trtVOZEqdacy0CM/SJxjRA8HQ7/0pjtqjOvcpZU - IB6z7IbZLH7krqJY4ZHS6gFvH3B7YOksaQsQL0x4GsdYY4mGUj/18Dzzw2YscUjs - HCOuN9zwAxEIztSQFFZ8vShbpk73fu80X5qRoCQ9708+sKdO92oDY9oZBQkcUl1T - qhpSdApN9mJl2n+uHfSDy63YynhT/bMMrh0AfZjB4ssX9jNkH2knS/FVFUjFUHVh - 6boXr0q9xdxt64onx8BrpWOBCqqXjRWUR2n/+y+zw+YgjqUWjpVmsQoF7wQQ3xo6 - Yb8y2WguTG9K6m9rS96dOtkXWJgZOVYZ5zlRqdbGZzlfei1890QfnRsNJQhhwkLq - CJV5bhy6AGZxk9JK/RW33g//i2GDfUx4HptRPEgGWu3ZdQskKwyZB7dc6NMtT2as - tOP6z/wgLIPVlLJEY0jXHkmbGf7Oj9JpBSCQBz57rmZunsTgy/jDIuL6mzeuVdYN - lVHqVao23aTZRPaCmwYqWW254oCKaeE7X8nRaQF+9L2nK4YbUf0+KbDGhjnQy8Qg - K1cQt51NcWsM28jNV6Puww7MS+K0NaMjr1fTHdomfHI27C0Dr1e85BWkDnesLqtw - 2s5S/8KdYMdBLuzyfT4UQkYTmtxibRXQR9+TxDmNQ/luMuFTCowgGfebAMOCrwU7 - NxrgSyuTmAC1Je1glSMMQghHwBCUB2BUCn/vFlMwHdl1waKrUpRaKQRI3iPhMjMw - 91Fsv5cUc6uD2pO7vb+vOm2O7+i08KtBpttjk+ANDJjxiGT0V/omlh40T80vN0h6 - yk8ZNTq8MqqvLMyH2wKqmmEjll73AWkHATLawRD3ckmlEF+ywc6J91CAYXokWuHc - N7CBL3vRhEJppZ3rmKNw3ani+ThQLTqnGxzxuB+P5IBO6RGXvjYfiUC3Nb0o1Q6X - +QD5BZlvVkklG4bwRdcn87wSlarA8T/nqlZ388ajNaE1Y2+zyJnJyOUEk3nLcgI8 - ovaVF/G3PG4yhPR+oOgE7IdWwp+WFa15OF2iLn8ByQa3V8fsWczXHu/iXLyr0KKl - MJCR4bsCv2hcOFTlYSRMyBs+A9gXA9pT+ljv0g7/Z9BuFSmr6pRzgK/guk6WzoTt - m+TxDn1hEovo62KkhAyMtD1hbYO/5GDB6X8tI0YM0kRk8E+H8fuxl43uUE+y9B0X - 7Qmkf1Oym9x23S+372MiEa/avAWZTtHhhii37lWkKU+pkx+aiMrfJyozafx6cAaQ - Rxx5uv+8lXEZy4qNEXop7yKDz2agSd6XdZziSIO69BF3x6DMKZdBJtyc5V2RqibU - t80ziVK0IStJmNUPZ1DSMXiwN3yzkQ/bm9RH3x3PPvaVNjISHdl85wlDFc8FM2m0 - Q0RM40lj5XAEs1O8iBk5m9yCNMSKQLq5vOhmbygK3ILp4dBoYr6EGZjz+Nq4M+ws - n/dzdR62oCVuKYvVyJVUkmt4DGTo7Pi9ngjAdmLu/RLL8M0/MG9wbu2adT7c2ypj - HM3lUqm+KEf9CdpJBVj0RH5BDWKwDpWx6g6np+GoXsj9nkXYv5qxzVNwgpjTRHwH - xJE+1nFStBtiWunP6eqd8Fl99/jATgVU9ytp+Q+nnZPZn+KHCZEl3CF/TBKsNl7S - QUwdepNF80MDYFi2r685SqM6fvefur0sqyeDwsBOM4GBU88FH9GnWJhQqKVEmQH2 - PV/UzkCPpj0ngkQiQjGMQjOKmI6npljOWbIw7LrhggOnfFnP2iTO0B4aAx1h6Ppi - 3+jkrdJEuxB89f8P/W8ChtOw7s53YTwYtxmZ+/x0e1G4Nh8pPcFRFF2t/UHEav5v - s3CyH7reAIXDclHH46wbrczvcf6FzS+o8ypIRFAapamUhPqpksuIvyoUeQv8WW/Q - m2tFOPp9wJp/+GAEbuZTyTd/o7Cms3Zl0EOQB9tgqWyqWhasPd40/SCdeXzqpEMS - 5Io0tE0ohY9DzN96kn2+07FUSqOYInup3+EXUhCGF8K/i1dny6/o7ZxDjW5xsTdb - AZxd0UEdhvJtvtKhckLhICzImeLGrCUz/zuJBvTR08ir8Rm8kkAmHBn9/jf8+42J - X3TSTes7+k+DtZP6VL6RKhTAzFIEWLQZ+38nzGPfM0BUKf0sGW3wlWFQREU9k9QX - S/idPNOqdNHz/l0eUwf3/bjfAB6OqitPWYH23d6zMkMEgwx/gJmboOfiYu9FKvXJ - tvRgOHb6Rww8fUQhlDOhVupo0DFTIghdeXjeVn/CxIUO67Ns+PI5IB+/sw1KxTIp - kZjjft/l3+mSnyJVyqvzKyfA1WhaXLXJfcJyeGt/Y45RiYnkbSdcbuNhngn+NpZZ - SAcS4vyUqkDQ6RvWU+fww8EYxptNALt9hnc1wD+e8b3Gz2citRrLrc4AhiZwafp8 - gj4HtKBXFz3oX2vhHgubTLuiEKhGc2dYXL9Z2PlXOWZhauTO00iVYfbWPsdTRSvi - QmKIP9QVzDq6StfHxs2x47yxrHrEtCjsHjDz3d+r+p6i6O212EHlCQewaPfAieBp - lw01cJB1KwyeoYgTczQkz6hhM+fj1RBMNDqxTHBVb3GGNh1nxu+4UR+tgQG/V/Ot - M/1NE8+yeRktzukDX1toXfCFXvRL3ijriHliaivWww== - =4M3l - -----END PGP MESSAGE----- - - --WLjzd46aUAiOcuNXjWTJItBZonI56MuAk-- - """ - And there is EML file "Inbox/signed.eml" - """ - Subject: signed - From: Bridge Test - To: Internal Bridge - Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000 - - -----BEGIN PGP SIGNED MESSAGE----- - Hash: SHA256 - - secret - -----BEGIN PGP SIGNATURE----- - - iQEzBAEBCAAdFiEENaE2ZPemenI4pZah/SJcGo7SJWIFAl+cCAgACgkQ/SJcGo7S - JWKsOQf/YakNXkMNjZIu8Hf1WflxtiDXVzTugOicC05k5W64oIqSHt0xNaFKE37k - //3eDMWbHvqHKFVdg7qcLsVPeVBaW3bdZUiexGM24OiGgyEitufnHQLOtEDTound - JyH5nUeHpvpBKIIOJZNBDM0HsRYnwKwrOWk3N2VRwog4J8J3cmJ/f9bPWNI/0OPT - qmtVGRVg6Ge83nZn51Vof//jFzkO4wGYCsE0aF0Ywc7nISZuyKQzmu/qgmwzDG50 - PjpvIQ/ygisRPNdRlylXEqyoIDCQ+v0AnxhhwX/5dbt6xMuMMOxBrFSC94Zce1Vj - x2ssXlT4ONPnkI/YWwhtQPLU628IMg== - =GiS3 - -----END PGP SIGNATURE----- - """ - And there is EML file "Inbox/encrypted-signed.eml" - """ - Subject: encrypted and signed - From: Bridge Test - To: Internal Bridge - Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000 - - -----BEGIN PGP MESSAGE----- - - hQEMA7hGUUsYs0fEAQf/dppHciWIf+o4l0gEfHeyHV/HVhG4es0aVQYrwFQlSWVx - estMuyLBSMfrsQXLago7Q9ZNo/XnKszzprCXxxYH52hAg64oAsjKB3jgRmVizs8b - 8lj0BRf003wUluS/0msV9SiEZBGeL8jGq6Te9vaM8OHHhIVzVjGnRdTSC0jBE6cS - vy8IBHXYe0LfdZiPojPDPGQdSej+H3uu7eZGBvVHTDeQLPDel4k7Ykdr0qlNXs6O - 5XpM5YG4w+t0aG+YROPH+BUj8PpPojQ/lrv/yFISTRbHlEd8N50w8BNTnBet+9Vm - oPcyvN+RQxBlvRuPpDjUmREvmtObKZV6+m6gocemx9LAzQEeVLcpjO/hJhl8gX72 - MNz3McU7aXf5sSoOPdHDNx8T2NON/2bwG5FE+PRMuVywTKhCB7o8VAsJpGMQ8xRM - 5WCNhow0AI7kni8yZA+GbvspnJWfit9tCTR5MIFHCSH9J3kJJnWkxQSN04GGpBcd - n43GWn7O7ufA4lMMZiGXMdi/J1iV9waAsIfMPk29BMq6xK0/jJYdHqQS+vNsSnF5 - xL/Ir4RYq4SFFA06A/E7HpXr2ruZhBQCkzaIIdrVJR/Lp2VLJIVulTBQK8y2AFtj - JeeKS0kIuC/7UPF2O624kwNr8dmIhDJYusFs6ZeED/nAKwDO/vP2CSwVC3sUjn3N - u+sWqQUTxSmjhRVf9b0+VyTh0mXCovJQXomL6Zz6lxXuJqqzELIOfCxYD1z9GwTG - cT08Aa2eEpf3agdLCTxvjO3iq9FksMHvIN+LSCQ6Pw+aTByjrk0oMmvGbANAogTk - yrplG/iRVlmq0p/Cfl5UEjKqT/nt5j9zbpeuYXmhjiBT9SBE07oUVLY1VT7ihcY= - =qYnL - -----END PGP MESSAGE----- - """ - - Scenario: Import encrypted - Given there is skip encrypted messages set to "false" - When user "user" imports local files - Then progress result is "OK" - And transfer failed for 0 messages - And transfer exported 5 messages - And transfer imported 5 messages - And transfer skipped 0 messages - And API mailbox "INBOX" for "user" has messages - | from | to | subject | - | bridgetest@pm.test | test@protonmail.com | clear | - | bridgetest@pm.test | test@protonmail.com | encrypted | - | bridgetest@pm.test | test@protonmail.com | encrypted mime | - | bridgetest@pm.test | test@protonmail.com | signed | - | bridgetest@pm.test | test@protonmail.com | encrypted and signed | - - Scenario: Skip encrypted - Given there is skip encrypted messages set to "true" - When user "user" imports local files - Then progress result is "OK" - And transfer failed for 0 messages - And transfer exported 5 messages - And transfer imported 2 messages - And transfer skipped 3 messages - And API mailbox "INBOX" for "user" has messages - | from | to | subject | - | bridgetest@pm.test | test@protonmail.com | clear | - | bridgetest@pm.test | test@protonmail.com | signed | diff --git a/test/features/ie/transfer/import_export.feature b/test/features/ie/transfer/import_export.feature deleted file mode 100644 index 33cd9f36..00000000 --- a/test/features/ie/transfer/import_export.feature +++ /dev/null @@ -1,49 +0,0 @@ -Feature: Import-Export app - Background: - Given there is connected user "user" - And there is "user" with mailbox "Folders/Foo" - And there is "user" with mailbox "Folders/Bar" - - Scenario: EML -> PM -> EML - Given there are EML files - | file | from | to | subject | time | - | Inbox/hello.eml | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 | - | Foo/one.eml | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 | - | Foo/two.eml | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - | Sub/Foo/three.eml | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | - - When user "user" imports local files - Then progress result is "OK" - And transfer failed for 0 messages - And transfer imported 4 messages - - When user "user" exports to EML files - Then progress result is "OK" - And transfer failed for 0 messages - # Every message is also in All Mail. - And transfer imported 8 messages - - And exported messages match the original ones - - Scenario: MBOX -> PM -> MBOX - Given there is MBOX file "Inbox.mbox" with messages - | from | to | subject | time | - | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 | - And there is MBOX file "Foo.mbox" with messages - | from | to | subject | time | - | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 | - | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | - - When user "user" imports local files - Then progress result is "OK" - And transfer failed for 0 messages - And transfer imported 4 messages - - When user "user" exports to MBOX files - Then progress result is "OK" - And transfer failed for 0 messages - # Every message is also in All Mail. - And transfer imported 8 messages - - And exported messages match the original ones diff --git a/test/features/ie/transfer/import_imap.feature b/test/features/ie/transfer/import_imap.feature deleted file mode 100644 index 47e626d7..00000000 --- a/test/features/ie/transfer/import_imap.feature +++ /dev/null @@ -1,80 +0,0 @@ -Feature: Import from IMAP server - Background: - Given there is connected user "user" - And there is "user" with mailbox "Folders/Foo" - And there is "user" with mailbox "Folders/Bar" - And there are IMAP mailboxes - | name | - | Inbox | - | Foo | - | Broken | - And there are IMAP messages - | mailbox | seqnum | uid | from | to | subject | time | - | Foo | 1 | 12 | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 | - | Foo | 2 | 14 | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - | Foo | 3 | 15 | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | - And there is IMAP message in mailbox "Inbox" with seq 1, uid 42, time "2020-01-01T12:34:56" and subject "hello" - """ - Subject: hello - From: Bridge Test - To: Internal Bridge - Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000 - - hello - - """ - - Scenario: Import all - When user "user" imports remote messages - Then progress result is "OK" - And transfer exported 4 messages - And transfer imported 4 messages - And transfer failed for 0 messages - And API mailbox "INBOX" for "user" has messages - | from | to | subject | - | bridgetest@pm.test | test@protonmail.com | hello | - And API mailbox "Folders/Foo" for "user" has messages - | from | to | subject | - | foo@example.com | bridgetest@protonmail.com | one | - | bar@example.com | bridgetest@protonmail.com | two | - | bar@example.com | bridgetest@protonmail.com | three | - - Scenario: Import only Foo to Bar with time limit - When user "user" imports remote messages with rules - | source | target | from | to | - | Foo | Bar | 2020-01-01T12:10:00 | 2020-01-01T13:00:00 | - Then progress result is "OK" - And transfer exported 2 messages - And transfer imported 2 messages - And transfer failed for 0 messages - And API mailbox "Folders/Bar" for "user" has messages - | from | to | subject | - | bar@example.com | bridgetest@protonmail.com | two | - | bar@example.com | bridgetest@protonmail.com | three | - - # Note we need to have message which we can parse and use in go-imap - # but which has problem on our side. Used example with missing boundary - # is real example which we want to solve one day. Probabl this test - # can be removed once we import any time of message or switch is to - # something we will never allow. - Scenario: Import broken message - Given there is IMAP message in mailbox "Broken" with seq 1, uid 42, time "2020-01-01T12:34:56" and subject "broken" - """ - Subject: missing boundary end - Content-Type: multipart/related; boundary=boundary - - --boundary - Content-Disposition: inline - Content-Transfer-Encoding: quoted-printable - Content-Type: text/plain; charset=utf-8 - - body - - """ - When user "user" imports remote messages with rules - | source | target | - | Broken | Foo | - Then progress result is "OK" - And transfer exported 1 messages - And transfer imported 0 messages - And transfer failed for 1 messages diff --git a/test/features/ie/transfer/import_mbox.feature b/test/features/ie/transfer/import_mbox.feature deleted file mode 100644 index 4ad66dcb..00000000 --- a/test/features/ie/transfer/import_mbox.feature +++ /dev/null @@ -1,65 +0,0 @@ -Feature: Import from MBOX files - Background: - Given there is connected user "user" - And there is "user" with mailbox "Folders/Foo" - And there is "user" with mailbox "Folders/Bar" - And there is MBOX file "Foo.mbox" with messages - | from | to | subject | time | - | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 | - | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 | - And there is MBOX file "Sub/Foo.mbox" with messages - | from | to | subject | time | - | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 | - And there is MBOX file "Inbox.mbox" - """ - From bridgetest@pm.test Thu Feb 20 20:20:20 2020 - Subject: hello - From: Bridge Test - To: Internal Bridge - Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000 - - hello - - """ - - Scenario: Import all - When user "user" imports local files - Then progress result is "OK" - And transfer exported 4 messages - And transfer imported 4 messages - And transfer failed for 0 messages - And API mailbox "INBOX" for "user" has messages - | from | to | subject | - | bridgetest@pm.test | test@protonmail.com | hello | - And API mailbox "Folders/Foo" for "user" has messages - | from | to | subject | - | foo@example.com | bridgetest@protonmail.com | one | - | bar@example.com | bridgetest@protonmail.com | two | - | bar@example.com | bridgetest@protonmail.com | three | - - Scenario: Import only Foo to Bar with time limit - When user "user" imports local files with rules - | source | target | from | to | - | Foo | Bar | 2020-01-01T12:10:00 | 2020-01-01T13:00:00 | - Then progress result is "OK" - And transfer exported 2 messages - And transfer imported 2 messages - And transfer failed for 0 messages - And API mailbox "Folders/Bar" for "user" has messages - | from | to | subject | - | bar@example.com | bridgetest@protonmail.com | two | - | bar@example.com | bridgetest@protonmail.com | three | - - Scenario: Import broken message - Given there is MBOX file "Broken.mbox" - """ - From bridgetest@pm.test Thu Feb 20 20:20:20 2020 - Content-type: multipart/mixed - """ - When user "user" imports local files with rules - | source | target | - | Broken | Foo | - Then progress result is "OK" - And transfer exported 1 messages - And transfer imported 0 messages - And transfer failed for 1 messages diff --git a/test/features/ie/transfer/import_sent.feature b/test/features/ie/transfer/import_sent.feature deleted file mode 100644 index 3170dd64..00000000 --- a/test/features/ie/transfer/import_sent.feature +++ /dev/null @@ -1,98 +0,0 @@ -Feature: Import to sent - Background: - Given there is connected user "user" - And there is "user" with mailbox "Labels/label" - And there is EML file "Sent/one.eml" - """ - Subject: one - From: Foo - To: Bridge Test - Message-ID: one.integrationtest - - one - - """ - And there is EML file "Sent/two.eml" - """ - Subject: two - From: Bar - To: Bridge Test - Message-ID: two.integrationtest - - two - - """ - - Scenario: Import sent only - When user "user" imports local files - Then progress result is "OK" - And transfer exported 2 messages - And transfer imported 2 messages - And transfer failed for 0 messages - And API mailbox "INBOX" for "user" has 0 message - And API mailbox "Sent" for "user" has messages - | from | to | subject | - | foo@example.com | bridgetest@pm.test | one | - | bar@example.com | bridgetest@pm.test | two | - - # Messages imported to label only are added automatically to Archive folder. - # Then it depends on the order: if the message is first imported to Sent - # folder and later to that label with importing to Archive, message will not - # be in Sent but Archive. The order is semi-random for the big messages, - # e.g., it will do alphabetical order of mailboxes, but for under ten small - # messages the order is random every time (because we are importing in - # batches of up to ten messages and iterating through map we use to collect - # messages is random). So we cannot for this test ensure the same output - # every time. - @ignore-live - Scenario: Import to sent and custom label - And there is EML file "Label/one.eml" - """ - Subject: one - From: Foo - To: Bridge Test - Message-ID: one.integrationtest - - one - - """ - When user "user" imports local files - Then progress result is "OK" - And transfer exported 3 messages - And transfer imported 3 messages - And transfer failed for 0 messages - # We had an issue that moving message to Sent automatically added - # the message also into Inbox if the message was in some custom label. - And API mailbox "INBOX" for "user" has 0 message - And API mailbox "Labels/label" for "user" has messages - | from | to | subject | - | foo@example.com | bridgetest@pm.test | one | - And API mailbox "Sent" for "user" has messages - | from | to | subject | - | foo@example.com | bridgetest@pm.test | one | - | bar@example.com | bridgetest@pm.test | two | - - Scenario: Import to sent and inbox is in both mailboxes - And there is EML file "Inbox/one.eml" - """ - Subject: one - From: Foo - To: Bridge Test - Message-ID: one.integrationtest - Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000 - - one - - """ - When user "user" imports local files - Then progress result is "OK" - And transfer exported 3 messages - And transfer imported 3 messages - And transfer failed for 0 messages - And API mailbox "INBOX" for "user" has messages - | from | to | subject | - | foo@example.com | bridgetest@pm.test | one | - And API mailbox "Sent" for "user" has messages - | from | to | subject | - | foo@example.com | bridgetest@pm.test | one | - | bar@example.com | bridgetest@pm.test | two | diff --git a/test/features/ie/users/delete.feature b/test/features/ie/users/delete.feature deleted file mode 100644 index 34ffe4b0..00000000 --- a/test/features/ie/users/delete.feature +++ /dev/null @@ -1,20 +0,0 @@ -Feature: Delete user - Scenario: Deleting connected user - Given there is connected user "user" - When user deletes "user" - Then last response is "OK" - - Scenario: Deleting connected user with cache - Given there is connected user "user" - When user deletes "user" with cache - Then last response is "OK" - - Scenario: Deleting disconnected user - Given there is disconnected user "user" - When user deletes "user" - Then last response is "OK" - - Scenario: Deleting disconnected user with cache - Given there is disconnected user "user" - When user deletes "user" with cache - Then last response is "OK" diff --git a/test/features/ie/users/login.feature b/test/features/ie/users/login.feature deleted file mode 100644 index 284127d6..00000000 --- a/test/features/ie/users/login.feature +++ /dev/null @@ -1,58 +0,0 @@ -Feature: Login for the first time - Scenario: Normal login - Given there is user "user" - When "user" logs in - Then last response is "OK" - And "user" is connected - - @ignore-live - Scenario: Login with bad username - When "user" logs in with bad password - Then last response is "failed to login: Incorrect login credentials. Please try again" - - @ignore-live - Scenario: Login with bad password - Given there is user "user" - When "user" logs in with bad password - Then last response is "failed to login: Incorrect login credentials. Please try again" - - Scenario: Login without internet connection - Given there is no internet connection - When "user" logs in - Then last response is "failed to login: no internet connection" - - @ignore-live - Scenario: Login user with 2FA - Given there is user "user2fa" - When "user2fa" logs in - Then last response is "OK" - And "user2fa" is connected - - Scenario: Login user with capital letters in address - Given there is user "userAddressWithCapitalLetter" - When "userAddressWithCapitalLetter" logs in - Then last response is "OK" - And "userAddressWithCapitalLetter" is connected - - Scenario: Login user with more addresses - Given there is user "userMoreAddresses" - When "userMoreAddresses" logs in - Then last response is "OK" - And "userMoreAddresses" is connected - - @ignore-live - Scenario: Login user with disabled primary address - Given there is user "userDisabledPrimaryAddress" - When "userDisabledPrimaryAddress" logs in - Then last response is "OK" - And "userDisabledPrimaryAddress" is connected - - Scenario: Login two users - Given there is user "user" - And there is user "userMoreAddresses" - When "user" logs in - Then last response is "OK" - And "user" is connected - When "userMoreAddresses" logs in - Then last response is "OK" - And "userMoreAddresses" is connected diff --git a/test/features/ie/users/relogin.feature b/test/features/ie/users/relogin.feature deleted file mode 100644 index d55e3eea..00000000 --- a/test/features/ie/users/relogin.feature +++ /dev/null @@ -1,12 +0,0 @@ -Feature: Re-login - Scenario: Re-login with connected user - Given there is connected user "user" - When "user" logs in - Then last response is "failed to finish login: user is already connected" - And "user" is connected - - Scenario: Re-login with disconnected user - Given there is disconnected user "user" - When "user" logs in - Then last response is "OK" - And "user" is connected diff --git a/test/features/bridge/imap/auth.feature b/test/features/imap/auth.feature similarity index 100% rename from test/features/bridge/imap/auth.feature rename to test/features/imap/auth.feature diff --git a/test/features/bridge/imap/idle/basic.feature b/test/features/imap/idle/basic.feature similarity index 100% rename from test/features/bridge/imap/idle/basic.feature rename to test/features/imap/idle/basic.feature diff --git a/test/features/bridge/imap/idle/two_users.feature b/test/features/imap/idle/two_users.feature similarity index 100% rename from test/features/bridge/imap/idle/two_users.feature rename to test/features/imap/idle/two_users.feature diff --git a/test/features/bridge/imap/mailbox/create.feature b/test/features/imap/mailbox/create.feature similarity index 100% rename from test/features/bridge/imap/mailbox/create.feature rename to test/features/imap/mailbox/create.feature diff --git a/test/features/bridge/imap/mailbox/delete.feature b/test/features/imap/mailbox/delete.feature similarity index 100% rename from test/features/bridge/imap/mailbox/delete.feature rename to test/features/imap/mailbox/delete.feature diff --git a/test/features/bridge/imap/mailbox/info.feature b/test/features/imap/mailbox/info.feature similarity index 100% rename from test/features/bridge/imap/mailbox/info.feature rename to test/features/imap/mailbox/info.feature diff --git a/test/features/bridge/imap/mailbox/list.feature b/test/features/imap/mailbox/list.feature similarity index 100% rename from test/features/bridge/imap/mailbox/list.feature rename to test/features/imap/mailbox/list.feature diff --git a/test/features/bridge/imap/mailbox/rename.feature b/test/features/imap/mailbox/rename.feature similarity index 100% rename from test/features/bridge/imap/mailbox/rename.feature rename to test/features/imap/mailbox/rename.feature diff --git a/test/features/bridge/imap/mailbox/select.feature b/test/features/imap/mailbox/select.feature similarity index 100% rename from test/features/bridge/imap/mailbox/select.feature rename to test/features/imap/mailbox/select.feature diff --git a/test/features/bridge/imap/mailbox/status.feature b/test/features/imap/mailbox/status.feature similarity index 100% rename from test/features/bridge/imap/mailbox/status.feature rename to test/features/imap/mailbox/status.feature diff --git a/test/features/bridge/imap/message/copy.feature b/test/features/imap/message/copy.feature similarity index 100% rename from test/features/bridge/imap/message/copy.feature rename to test/features/imap/message/copy.feature diff --git a/test/features/bridge/imap/message/create.feature b/test/features/imap/message/create.feature similarity index 100% rename from test/features/bridge/imap/message/create.feature rename to test/features/imap/message/create.feature diff --git a/test/features/bridge/imap/message/delete.feature b/test/features/imap/message/delete.feature similarity index 100% rename from test/features/bridge/imap/message/delete.feature rename to test/features/imap/message/delete.feature diff --git a/test/features/bridge/imap/message/delete_from_trash.feature b/test/features/imap/message/delete_from_trash.feature similarity index 100% rename from test/features/bridge/imap/message/delete_from_trash.feature rename to test/features/imap/message/delete_from_trash.feature diff --git a/test/features/bridge/imap/message/fetch.feature b/test/features/imap/message/fetch.feature similarity index 100% rename from test/features/bridge/imap/message/fetch.feature rename to test/features/imap/message/fetch.feature diff --git a/test/features/bridge/imap/message/fetch_header.feature b/test/features/imap/message/fetch_header.feature similarity index 100% rename from test/features/bridge/imap/message/fetch_header.feature rename to test/features/imap/message/fetch_header.feature diff --git a/test/features/bridge/imap/message/import.feature b/test/features/imap/message/import.feature similarity index 100% rename from test/features/bridge/imap/message/import.feature rename to test/features/imap/message/import.feature diff --git a/test/features/bridge/imap/message/move.feature b/test/features/imap/message/move.feature similarity index 100% rename from test/features/bridge/imap/message/move.feature rename to test/features/imap/message/move.feature diff --git a/test/features/bridge/imap/message/move_local_folder.feature b/test/features/imap/message/move_local_folder.feature similarity index 100% rename from test/features/bridge/imap/message/move_local_folder.feature rename to test/features/imap/message/move_local_folder.feature diff --git a/test/features/bridge/imap/message/move_without_support.feature b/test/features/imap/message/move_without_support.feature similarity index 100% rename from test/features/bridge/imap/message/move_without_support.feature rename to test/features/imap/message/move_without_support.feature diff --git a/test/features/bridge/imap/message/search.feature b/test/features/imap/message/search.feature similarity index 100% rename from test/features/bridge/imap/message/search.feature rename to test/features/imap/message/search.feature diff --git a/test/features/bridge/imap/message/update.feature b/test/features/imap/message/update.feature similarity index 100% rename from test/features/bridge/imap/message/update.feature rename to test/features/imap/message/update.feature diff --git a/test/features/bridge/imap/message/update_spam.feature b/test/features/imap/message/update_spam.feature similarity index 100% rename from test/features/bridge/imap/message/update_spam.feature rename to test/features/imap/message/update_spam.feature diff --git a/test/features/bridge/imap/user_agent.feature b/test/features/imap/user_agent.feature similarity index 100% rename from test/features/bridge/imap/user_agent.feature rename to test/features/imap/user_agent.feature diff --git a/test/features/no_internet.feature b/test/features/no_internet.feature new file mode 100644 index 00000000..5abb1841 --- /dev/null +++ b/test/features/no_internet.feature @@ -0,0 +1,30 @@ +Feature: Servers are closed when no internet + + Scenario: All connection are closed and then restored multiple times + Given there is connected user "user" + And there is IMAP client "i1" logged in as "user" + And there is SMTP client "s1" logged in as "user" + When there is no internet connection + And 1 second pass + Then IMAP client "i1" is logged out + And SMTP client "s1" is logged out + Given the internet connection is restored + And 1 second pass + And there is IMAP client "i2" logged in as "user" + And there is SMTP client "s2" logged in as "user" + When IMAP client "i2" gets info of "INBOX" + When SMTP client "s2" sends "HELO example.com" + Then IMAP response to "i2" is "OK" + Then SMTP response to "s2" is "OK" + When there is no internet connection + And 1 second pass + Then IMAP client "i2" is logged out + And SMTP client "s2" is logged out + Given the internet connection is restored + And 1 second pass + And there is IMAP client "i3" logged in as "user" + And there is SMTP client "s3" logged in as "user" + When IMAP client "i3" gets info of "INBOX" + When SMTP client "s3" sends "HELO example.com" + Then IMAP response to "i3" is "OK" + Then SMTP response to "s3" is "OK" diff --git a/test/features/bridge/smtp/auth.feature b/test/features/smtp/auth.feature similarity index 100% rename from test/features/bridge/smtp/auth.feature rename to test/features/smtp/auth.feature diff --git a/test/features/bridge/smtp/init.feature b/test/features/smtp/init.feature similarity index 100% rename from test/features/bridge/smtp/init.feature rename to test/features/smtp/init.feature diff --git a/test/features/bridge/smtp/send/bcc.feature b/test/features/smtp/send/bcc.feature similarity index 100% rename from test/features/bridge/smtp/send/bcc.feature rename to test/features/smtp/send/bcc.feature diff --git a/test/features/bridge/smtp/send/embedded_message.feature b/test/features/smtp/send/embedded_message.feature similarity index 100% rename from test/features/bridge/smtp/send/embedded_message.feature rename to test/features/smtp/send/embedded_message.feature diff --git a/test/features/bridge/smtp/send/failures.feature b/test/features/smtp/send/failures.feature similarity index 100% rename from test/features/bridge/smtp/send/failures.feature rename to test/features/smtp/send/failures.feature diff --git a/test/features/bridge/smtp/send/html.feature b/test/features/smtp/send/html.feature similarity index 100% rename from test/features/bridge/smtp/send/html.feature rename to test/features/smtp/send/html.feature diff --git a/test/features/bridge/smtp/send/html_att.feature b/test/features/smtp/send/html_att.feature similarity index 100% rename from test/features/bridge/smtp/send/html_att.feature rename to test/features/smtp/send/html_att.feature diff --git a/test/features/bridge/smtp/send/mixed_case.feature b/test/features/smtp/send/mixed_case.feature similarity index 100% rename from test/features/bridge/smtp/send/mixed_case.feature rename to test/features/smtp/send/mixed_case.feature diff --git a/test/features/bridge/smtp/send/plain.feature b/test/features/smtp/send/plain.feature similarity index 100% rename from test/features/bridge/smtp/send/plain.feature rename to test/features/smtp/send/plain.feature diff --git a/test/features/bridge/smtp/send/plain_att.feature b/test/features/smtp/send/plain_att.feature similarity index 100% rename from test/features/bridge/smtp/send/plain_att.feature rename to test/features/smtp/send/plain_att.feature diff --git a/test/features/bridge/smtp/send/same_message.feature b/test/features/smtp/send/same_message.feature similarity index 100% rename from test/features/bridge/smtp/send/same_message.feature rename to test/features/smtp/send/same_message.feature diff --git a/test/features/bridge/smtp/send/send_append.feature b/test/features/smtp/send/send_append.feature similarity index 100% rename from test/features/bridge/smtp/send/send_append.feature rename to test/features/smtp/send/send_append.feature diff --git a/test/features/bridge/smtp/send/two_messages.feature b/test/features/smtp/send/two_messages.feature similarity index 100% rename from test/features/bridge/smtp/send/two_messages.feature rename to test/features/smtp/send/two_messages.feature diff --git a/test/features/bridge/start.feature b/test/features/start.feature similarity index 100% rename from test/features/bridge/start.feature rename to test/features/start.feature diff --git a/test/features/bridge/users/addressmode.feature b/test/features/users/addressmode.feature similarity index 100% rename from test/features/bridge/users/addressmode.feature rename to test/features/users/addressmode.feature diff --git a/test/features/bridge/users/delete.feature b/test/features/users/delete.feature similarity index 100% rename from test/features/bridge/users/delete.feature rename to test/features/users/delete.feature diff --git a/test/features/bridge/users/login.feature b/test/features/users/login.feature similarity index 100% rename from test/features/bridge/users/login.feature rename to test/features/users/login.feature diff --git a/test/features/bridge/users/relogin.feature b/test/features/users/relogin.feature similarity index 100% rename from test/features/bridge/users/relogin.feature rename to test/features/users/relogin.feature diff --git a/test/features/bridge/users/sync.feature b/test/features/users/sync.feature similarity index 100% rename from test/features/bridge/users/sync.feature rename to test/features/users/sync.feature diff --git a/test/liveapi/controller.go b/test/liveapi/controller.go index 774c3b33..88ff66df 100644 --- a/test/liveapi/controller.go +++ b/test/liveapi/controller.go @@ -38,7 +38,7 @@ type Controller struct { noInternetConnection bool } -func NewController(_ string) (*Controller, pmapi.Manager) { +func NewController() (*Controller, pmapi.Manager) { controller := &Controller{ log: logrus.WithField("pkg", "live-controller"), lock: &sync.RWMutex{}, diff --git a/test/transfer_actions_test.go b/test/transfer_actions_test.go deleted file mode 100644 index c2a99a2b..00000000 --- a/test/transfer_actions_test.go +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge.Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package tests - -import ( - "fmt" - "time" - - "github.com/ProtonMail/proton-bridge/internal/transfer" - "github.com/cucumber/godog" -) - -func TransferActionsFeatureContext(s *godog.ScenarioContext) { - s.Step(`^user "([^"]*)" imports local files$`, userImportsLocalFiles) - s.Step(`^user "([^"]*)" imports local files with rules$`, userImportsLocalFilesWithRules) - s.Step(`^user "([^"]*)" imports local files to address "([^"]*)"$`, userImportsLocalFilesToAddress) - s.Step(`^user "([^"]*)" imports local files to address "([^"]*)" with rules$`, userImportsLocalFilesToAddressWithRules) - s.Step(`^user "([^"]*)" imports remote messages$`, userImportsRemoteMessages) - s.Step(`^user "([^"]*)" imports remote messages with rules$`, userImportsRemoteMessagesWithRules) - s.Step(`^user "([^"]*)" imports remote messages to address "([^"]*)"$`, userImportsRemoteMessagesToAddress) - s.Step(`^user "([^"]*)" imports remote messages to address "([^"]*)" with rules$`, userImportsRemoteMessagesToAddressWithRules) - s.Step(`^user "([^"]*)" exports to EML files$`, userExportsToEMLFiles) - s.Step(`^user "([^"]*)" exports to EML files with rules$`, userExportsToEMLFilesWithRules) - s.Step(`^user "([^"]*)" exports address "([^"]*)" to EML files$`, userExportsAddressToEMLFiles) - s.Step(`^user "([^"]*)" exports address "([^"]*)" to EML files with rules$`, userExportsAddressToEMLFilesWithRules) - s.Step(`^user "([^"]*)" exports to MBOX files$`, userExportsToMBOXFiles) - s.Step(`^user "([^"]*)" exports to MBOX files with rules$`, userExportsToMBOXFilesWithRules) - s.Step(`^user "([^"]*)" exports address "([^"]*)" to MBOX files$`, userExportsAddressToMBOXFiles) - s.Step(`^user "([^"]*)" exports address "([^"]*)" to MBOX files with rules$`, userExportsAddressToMBOXFilesWithRules) -} - -// Local import. - -func userImportsLocalFiles(bddUserID string) error { - return userImportsLocalFilesToAddressWithRules(bddUserID, "", nil) -} - -func userImportsLocalFilesWithRules(bddUserID string, rules *godog.Table) error { - return userImportsLocalFilesToAddressWithRules(bddUserID, "", rules) -} - -func userImportsLocalFilesToAddress(bddUserID, bddAddressID string) error { - return userImportsLocalFilesToAddressWithRules(bddUserID, bddAddressID, nil) -} - -func userImportsLocalFilesToAddressWithRules(bddUserID, bddAddressID string, rules *godog.Table) error { - return doTransfer(bddUserID, bddAddressID, rules, func(username, address string) (*transfer.Transfer, error) { - path := ctx.GetTransferLocalRootForImport() - return ctx.GetImportExport().GetLocalImporter(username, address, path) - }) -} - -// Remote import. - -func userImportsRemoteMessages(bddUserID string) error { - return userImportsRemoteMessagesToAddressWithRules(bddUserID, "", nil) -} - -func userImportsRemoteMessagesWithRules(bddUserID string, rules *godog.Table) error { - return userImportsRemoteMessagesToAddressWithRules(bddUserID, "", rules) -} - -func userImportsRemoteMessagesToAddress(bddUserID, bddAddressID string) error { - return userImportsRemoteMessagesToAddressWithRules(bddUserID, bddAddressID, nil) -} - -func userImportsRemoteMessagesToAddressWithRules(bddUserID, bddAddressID string, rules *godog.Table) error { - return doTransfer(bddUserID, bddAddressID, rules, func(username, address string) (*transfer.Transfer, error) { - imapServer := ctx.GetTransferRemoteIMAPServer() - return ctx.GetImportExport().GetRemoteImporter(username, address, imapServer.Username, imapServer.Password, imapServer.Host, imapServer.Port) - }) -} - -// EML export. - -func userExportsToEMLFiles(bddUserID string) error { - return userExportsAddressToEMLFilesWithRules(bddUserID, "", nil) -} - -func userExportsToEMLFilesWithRules(bddUserID string, rules *godog.Table) error { - return userExportsAddressToEMLFilesWithRules(bddUserID, "", rules) -} - -func userExportsAddressToEMLFiles(bddUserID, bddAddressID string) error { - return userExportsAddressToEMLFilesWithRules(bddUserID, bddAddressID, nil) -} - -func userExportsAddressToEMLFilesWithRules(bddUserID, bddAddressID string, rules *godog.Table) error { - return doTransfer(bddUserID, bddAddressID, rules, func(username, address string) (*transfer.Transfer, error) { - path := ctx.GetTransferLocalRootForExport() - return ctx.GetImportExport().GetEMLExporter(username, address, path) - }) -} - -// MBOX export. - -func userExportsToMBOXFiles(bddUserID string) error { - return userExportsAddressToMBOXFilesWithRules(bddUserID, "", nil) -} - -func userExportsToMBOXFilesWithRules(bddUserID string, rules *godog.Table) error { - return userExportsAddressToMBOXFilesWithRules(bddUserID, "", rules) -} - -func userExportsAddressToMBOXFiles(bddUserID, bddAddressID string) error { - return userExportsAddressToMBOXFilesWithRules(bddUserID, bddAddressID, nil) -} - -func userExportsAddressToMBOXFilesWithRules(bddUserID, bddAddressID string, rules *godog.Table) error { - return doTransfer(bddUserID, bddAddressID, rules, func(username, address string) (*transfer.Transfer, error) { - path := ctx.GetTransferLocalRootForExport() - return ctx.GetImportExport().GetMBOXExporter(username, address, path) - }) -} - -// Helpers. - -func doTransfer(bddUserID, bddAddressID string, rules *godog.Table, getTransferrer func(string, string) (*transfer.Transfer, error)) error { - account := ctx.GetTestAccountWithAddress(bddUserID, bddAddressID) - if account == nil { - return godog.ErrPending - } - transferrer, err := getTransferrer(account.Username(), account.Address()) - if err != nil { - return internalError(err, "failed to init transfer") - } - if err := setRules(transferrer, rules); err != nil { - return internalError(err, "failed to set rules") - } - transferrer.SetSkipEncryptedMessages(ctx.GetTransferSkipEncryptedMessages()) - progress := transferrer.Start() - ctx.SetTransferProgress(progress) - return nil -} - -func setRules(transferrer *transfer.Transfer, rules *godog.Table) error { - if rules == nil { - return nil - } - - transferrer.ResetRules() - - allSourceMailboxes, err := transferrer.SourceMailboxes() - if err != nil { - return internalError(err, "failed to get source mailboxes") - } - allTargetMailboxes, err := transferrer.TargetMailboxes() - if err != nil { - return internalError(err, "failed to get target mailboxes") - } - - head := rules.Rows[0].Cells - for _, row := range rules.Rows[1:] { - source := "" - target := "" - fromTime := int64(0) - toTime := int64(0) - for n, cell := range row.Cells { - switch head[n].Value { - case "source": - source = cell.Value - case "target": - target = cell.Value - case "from": - date, err := time.Parse(timeFormat, cell.Value) - if err != nil { - return internalError(err, "failed to parse from time") - } - fromTime = date.Unix() - case "to": - date, err := time.Parse(timeFormat, cell.Value) - if err != nil { - return internalError(err, "failed to parse to time") - } - toTime = date.Unix() - default: - return fmt.Errorf("unexpected column name: %s", head[n].Value) - } - } - - sourceMailbox, err := getMailboxByName(allSourceMailboxes, source) - if err != nil { - return internalError(err, "failed to match source mailboxes") - } - - // Empty target means the same as source. Useful for exports. - targetMailboxes := []transfer.Mailbox{} - if target == "" { - targetMailboxes = append(targetMailboxes, sourceMailbox) - } else { - targetMailbox, err := getMailboxByName(allTargetMailboxes, target) - if err != nil { - return internalError(err, "failed to match target mailboxes") - } - targetMailboxes = append(targetMailboxes, targetMailbox) - } - - if err := transferrer.SetRule(sourceMailbox, targetMailboxes, fromTime, toTime); err != nil { - return internalError(err, "failed to set rule") - } - } - return nil -} - -func getMailboxByName(mailboxes []transfer.Mailbox, name string) (transfer.Mailbox, error) { - for _, mailbox := range mailboxes { - if mailbox.Name == name { - return mailbox, nil - } - } - return transfer.Mailbox{}, fmt.Errorf("mailbox %s not found", name) -} diff --git a/test/transfer_checks_test.go b/test/transfer_checks_test.go deleted file mode 100644 index 253e8dee..00000000 --- a/test/transfer_checks_test.go +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge.Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package tests - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/ProtonMail/go-rfc5322" - "github.com/cucumber/godog" - "github.com/emersion/go-mbox" - "github.com/emersion/go-message" - "github.com/pkg/errors" - a "github.com/stretchr/testify/assert" -) - -func TransferChecksFeatureContext(s *godog.ScenarioContext) { - s.Step(`^progress result is "([^"]*)"$`, progressFinishedWith) - s.Step(`^transfer exported (\d+) messages$`, transferExportedNumberOfMessages) - s.Step(`^transfer imported (\d+) messages$`, transferImportedNumberOfMessages) - s.Step(`^transfer skipped (\d+) messages$`, transferSkippedNumberOfMessages) - s.Step(`^transfer failed for (\d+) messages$`, transferFailedForNumberOfMessages) - s.Step(`^transfer exported messages$`, transferExportedMessages) - s.Step(`^exported messages match the original ones$`, exportedMessagesMatchTheOriginalOnes) -} - -func progressFinishedWith(wantResponse string) error { - progress := ctx.GetTransferProgress() - // Wait till transport is finished. - updateCh := progress.GetUpdateChannel() - if updateCh != nil { - for range updateCh { - } - } - - err := progress.GetFatalError() - if wantResponse == "OK" { - a.NoError(ctx.GetTestingT(), err) - } else { - a.EqualError(ctx.GetTestingT(), err, wantResponse) - } - return ctx.GetTestingError() -} - -func transferExportedNumberOfMessages(wantCount int) error { - progress := ctx.GetTransferProgress() - counts := progress.GetCounts() - a.Equal(ctx.GetTestingT(), uint(wantCount), counts.Exported) - return ctx.GetTestingError() -} - -func transferImportedNumberOfMessages(wantCount int) error { - progress := ctx.GetTransferProgress() - counts := progress.GetCounts() - a.Equal(ctx.GetTestingT(), uint(wantCount), counts.Imported) - return ctx.GetTestingError() -} - -func transferSkippedNumberOfMessages(wantCount int) error { - progress := ctx.GetTransferProgress() - counts := progress.GetCounts() - a.Equal(ctx.GetTestingT(), uint(wantCount), counts.Skipped) - return ctx.GetTestingError() -} - -func transferFailedForNumberOfMessages(wantCount int) error { - progress := ctx.GetTransferProgress() - failedMessages := progress.GetFailedMessages() - a.Equal(ctx.GetTestingT(), wantCount, len(failedMessages), "failed messages: %v", failedMessages) - return ctx.GetTestingError() -} - -func transferExportedMessages(messages *godog.Table) error { - expectedMessages := map[string][]MessageAttributes{} - - head := messages.Rows[0].Cells - for _, row := range messages.Rows[1:] { - folder := "" - msg := MessageAttributes{} - - for n, cell := range row.Cells { - switch head[n].Value { - case "folder": - folder = cell.Value - case "subject": - msg.subject = cell.Value - case "from": - msg.from = cell.Value - case "to": - msg.to = []string{cell.Value} - case "time": - date, err := time.Parse(timeFormat, cell.Value) - if err != nil { - return internalError(err, "failed to parse time") - } - msg.date = date.Unix() - default: - return fmt.Errorf("unexpected column name: %s", head[n].Value) - } - } - - expectedMessages[folder] = append(expectedMessages[folder], msg) - sort.Sort(BySubject(expectedMessages[folder])) - } - - exportRoot := ctx.GetTransferLocalRootForExport() - exportedMessages, err := readMessages(exportRoot) - if err != nil { - return errors.Wrap(err, "scanning exported messages") - } - - a.Equal(ctx.GetTestingT(), expectedMessages, exportedMessages) - return ctx.GetTestingError() -} - -func exportedMessagesMatchTheOriginalOnes() error { - importRoot := ctx.GetTransferLocalRootForImport() - exportRoot := ctx.GetTransferLocalRootForExport() - - importMessages, err := readMessages(importRoot) - if err != nil { - return errors.Wrap(err, "scanning messages for import") - } - exportMessages, err := readMessages(exportRoot) - if err != nil { - return errors.Wrap(err, "scanning exported messages") - } - delete(exportMessages, "All Mail") // Ignore All Mail. - - a.Equal(ctx.GetTestingT(), importMessages, exportMessages) - return ctx.GetTestingError() -} - -func readMessages(root string) (map[string][]MessageAttributes, error) { - files, err := ioutil.ReadDir(root) - if err != nil { - return nil, err - } - - messagesPerLabel := map[string][]MessageAttributes{} - for _, file := range files { - if !file.IsDir() { - fileReader, err := os.Open(filepath.Join(root, file.Name())) - if err != nil { - return nil, errors.Wrap(err, "opening file") - } - - if filepath.Ext(file.Name()) == ".eml" { - label := filepath.Base(root) - msg, err := readMessageAttributes(fileReader) - if err != nil { - return nil, err - } - messagesPerLabel[label] = append(messagesPerLabel[label], msg) - sort.Sort(BySubject(messagesPerLabel[label])) - } else if filepath.Ext(file.Name()) == ".mbox" { - label := strings.TrimSuffix(file.Name(), ".mbox") - mboxReader := mbox.NewReader(fileReader) - for { - msgReader, err := mboxReader.NextMessage() - if err == io.EOF { - break - } else if err != nil { - return nil, errors.Wrap(err, "reading next message") - } - msg, err := readMessageAttributes(msgReader) - if err != nil { - return nil, err - } - messagesPerLabel[label] = append(messagesPerLabel[label], msg) - } - sort.Sort(BySubject(messagesPerLabel[label])) - } - } else { - subfolderRoot := filepath.Join(root, file.Name()) - subfolderMessagesPerLabel, err := readMessages(subfolderRoot) - if err != nil { - return nil, err - } - for key, value := range subfolderMessagesPerLabel { - messagesPerLabel[key] = append(messagesPerLabel[key], value...) - sort.Sort(BySubject(messagesPerLabel[key])) - } - } - } - return messagesPerLabel, nil -} - -type MessageAttributes struct { - subject string - from string - to []string - date int64 -} - -func readMessageAttributes(fileReader io.Reader) (MessageAttributes, error) { - entity, err := message.Read(fileReader) - if err != nil { - return MessageAttributes{}, errors.Wrap(err, "reading file") - } - date, err := parseTime(entity.Header.Get("date")) - if err != nil { - return MessageAttributes{}, errors.Wrap(err, "parsing date") - } - from, err := parseAddress(entity.Header.Get("from")) - if err != nil { - return MessageAttributes{}, errors.Wrap(err, "parsing from") - } - to, err := parseAddresses(entity.Header.Get("to")) - if err != nil { - return MessageAttributes{}, errors.Wrap(err, "parsing to") - } - return MessageAttributes{ - subject: entity.Header.Get("subject"), - from: from, - to: to, - date: date.Unix(), - }, nil -} - -func parseTime(input string) (time.Time, error) { - for _, format := range []string{time.RFC1123, time.RFC1123Z} { - t, err := time.Parse(format, input) - if err == nil { - return t, nil - } - } - return time.Time{}, errors.New("Unrecognized time format") -} - -func parseAddresses(input string) ([]string, error) { - addresses, err := rfc5322.ParseAddressList(input) - if err != nil { - return nil, err - } - result := []string{} - for _, address := range addresses { - result = append(result, address.Address) - } - return result, nil -} - -func parseAddress(input string) (string, error) { - address, err := rfc5322.ParseAddressList(input) - if err != nil { - return "", err - } - return address[0].Address, nil -} - -// BySubject implements sort.Interface based on the subject field. -type BySubject []MessageAttributes - -func (a BySubject) Len() int { return len(a) } -func (a BySubject) Less(i, j int) bool { return a[i].subject < a[j].subject } -func (a BySubject) Swap(i, j int) { a[i], a[j] = a[j], a[i] } diff --git a/test/transfer_setup_test.go b/test/transfer_setup_test.go deleted file mode 100644 index 19b62a55..00000000 --- a/test/transfer_setup_test.go +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge.Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package tests - -import ( - "bytes" - "fmt" - "net/textproto" - "os" - "path/filepath" - "strconv" - "time" - - "github.com/ProtonMail/proton-bridge/pkg/message" - "github.com/cucumber/godog" - "github.com/cucumber/messages-go/v16" - "github.com/emersion/go-imap" - "github.com/emersion/go-mbox" -) - -func TransferSetupFeatureContext(s *godog.ScenarioContext) { - s.Step(`^there are EML files$`, thereAreEMLFiles) - s.Step(`^there is EML file "([^"]*)"$`, thereIsEMLFile) - s.Step(`^there is MBOX file "([^"]*)" with messages$`, thereIsMBOXFileWithMessages) - s.Step(`^there is MBOX file "([^"]*)"$`, thereIsMBOXFile) - s.Step(`^there are IMAP mailboxes$`, thereAreIMAPMailboxes) - s.Step(`^there are IMAP messages$`, thereAreIMAPMessages) - s.Step(`^there is IMAP message in mailbox "([^"]*)" with seq (\d+), uid (\d+), time "([^"]*)" and subject "([^"]*)"$`, thereIsIMAPMessage) - s.Step(`^there is skip encrypted messages set to "([^"]*)"$`, thereIsSkipEncryptedMessagesSetTo) -} - -func thereAreEMLFiles(messages *godog.Table) error { - head := messages.Rows[0].Cells - for _, row := range messages.Rows[1:] { - fileName := "" - for n, cell := range row.Cells { - switch head[n].Value { - case "file": - fileName = cell.Value - case "from", "to", "subject", "time", "body": - default: - return fmt.Errorf("unexpected column name: %s", head[n].Value) - } - } - - body := getBodyFromDataRow(head, row) - if err := createFile(fileName, body); err != nil { - return err - } - } - return nil -} - -func thereIsEMLFile(fileName string, message *godog.DocString) error { - return createFile(fileName, message.Content) -} - -func thereIsMBOXFileWithMessages(fileName string, messages *godog.Table) error { - mboxBuffer := &bytes.Buffer{} - mboxWriter := mbox.NewWriter(mboxBuffer) - - head := messages.Rows[0].Cells - for _, row := range messages.Rows[1:] { - from := "" - for n, cell := range row.Cells { - switch head[n].Value { - case "from": - from = cell.Value - case "to", "subject", "time", "body": - default: - return fmt.Errorf("unexpected column name: %s", head[n].Value) - } - } - - body := getBodyFromDataRow(head, row) - - messageWriter, err := mboxWriter.CreateMessage(from, time.Now()) - if err != nil { - return err - } - _, err = messageWriter.Write([]byte(body)) - if err != nil { - return err - } - } - - return createFile(fileName, mboxBuffer.String()) -} - -func thereIsMBOXFile(fileName string, messages *godog.DocString) error { - return createFile(fileName, messages.Content) -} - -func thereAreIMAPMailboxes(mailboxes *godog.Table) error { - imapServer := ctx.GetTransferRemoteIMAPServer() - head := mailboxes.Rows[0].Cells - for _, row := range mailboxes.Rows[1:] { - mailboxName := "" - for n, cell := range row.Cells { - switch head[n].Value { - case "name": - mailboxName = cell.Value - default: - return fmt.Errorf("unexpected column name: %s", head[n].Value) - } - } - imapServer.AddMailbox(mailboxName) - } - return nil -} - -func thereAreIMAPMessages(messages *godog.Table) (err error) { - imapServer := ctx.GetTransferRemoteIMAPServer() - head := messages.Rows[0].Cells - for _, row := range messages.Rows[1:] { - mailboxName := "" - date := time.Now() - subject := "" - seqNum := 0 - uid := 0 - for n, cell := range row.Cells { - switch head[n].Value { - case "mailbox": - mailboxName = cell.Value - case "uid": - uid, err = strconv.Atoi(cell.Value) - if err != nil { - return internalError(err, "failed to parse uid") - } - case "seqnum": - seqNum, err = strconv.Atoi(cell.Value) - if err != nil { - return internalError(err, "failed to parse seqnum") - } - case "time": - date, err = time.Parse(timeFormat, cell.Value) - if err != nil { - return internalError(err, "failed to parse time") - } - case "subject": - subject = cell.Value - case "from", "to", "body": - default: - return fmt.Errorf("unexpected column name: %s", head[n].Value) - } - } - - body := getBodyFromDataRow(head, row) - imapMessage, err := getIMAPMessage(seqNum, uid, date, subject, body) - if err != nil { - return err - } - imapServer.AddMessage(mailboxName, imapMessage) - } - return nil -} - -func thereIsIMAPMessage(mailboxName string, seqNum, uid int, dateValue, subject string, message *godog.DocString) error { - imapServer := ctx.GetTransferRemoteIMAPServer() - - date, err := time.Parse(timeFormat, dateValue) - if err != nil { - return internalError(err, "failed to parse time") - } - - imapMessage, err := getIMAPMessage(seqNum, uid, date, subject, message.Content) - if err != nil { - return err - } - imapServer.AddMessage(mailboxName, imapMessage) - - return nil -} - -func getBodyFromDataRow(head []*messages.PickleTableCell, row *messages.PickleTableRow) string { - body := "hello" - headers := textproto.MIMEHeader{} - headers.Set("Received", "by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000") - for n, cell := range row.Cells { - switch head[n].Value { - case "from": - headers.Set("from", cell.Value) - case "to": - headers.Set("to", cell.Value) - case "subject": - headers.Set("subject", cell.Value) - case "time": - date, err := time.Parse(timeFormat, cell.Value) - if err != nil { - panic(err) - } - headers.Set("date", date.Format(time.RFC1123)) - case "body": - body = cell.Value - } - } - - buffer := &bytes.Buffer{} - _ = message.WriteHeader(buffer, headers) - return buffer.String() + body + "\n\n" -} - -func getIMAPMessage(seqNum, uid int, date time.Time, subject, body string) (*imap.Message, error) { - reader := bytes.NewBufferString(body) - bodyStructure, err := message.NewBodyStructure(reader) - if err != nil { - return nil, internalError(err, "failed to parse body structure") - } - imapBodyStructure, err := bodyStructure.IMAPBodyStructure([]int{}) - if err != nil { - return nil, internalError(err, "failed to parse body structure") - } - bodySection, _ := imap.ParseBodySectionName("BODY[]") - - return &imap.Message{ - SeqNum: uint32(seqNum), - Uid: uint32(uid), - Size: uint32(len(body)), - Envelope: &imap.Envelope{ - Date: date, - Subject: subject, - }, - BodyStructure: imapBodyStructure, - Body: map[*imap.BodySectionName]imap.Literal{ - bodySection: bytes.NewBufferString(body), - }, - }, nil -} - -func createFile(fileName, body string) error { - root := ctx.GetTransferLocalRootForImport() - filePath := filepath.Join(root, fileName) - - dirPath := filepath.Dir(filePath) - err := os.MkdirAll(dirPath, os.ModePerm) - if err != nil { - return internalError(err, "failed to create dir") - } - - f, err := os.Create(filePath) - if err != nil { - return internalError(err, "failed to create file") - } - defer f.Close() //nolint - - _, err = f.WriteString(body) - return internalError(err, "failed to write to file") -} - -func thereIsSkipEncryptedMessagesSetTo(value string) error { - switch value { - case "true": - ctx.SetTransferSkipEncryptedMessages(true) - case "false": - ctx.SetTransferSkipEncryptedMessages(false) - default: - return fmt.Errorf("expected either true or false, was %v", value) - } - return nil -} diff --git a/utils/enums.sh b/utils/enums.sh deleted file mode 100644 index 1d4cc700..00000000 --- a/utils/enums.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2021 Proton Technologies AG -# -# This file is part of ProtonMail Bridge. -# -# ProtonMail Bridge is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ProtonMail Bridge is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with ProtonMail Bridge. If not, see . - - -# create QML JSON object from list of golang constants -# run this script and output line stored in `out.qml` insert to `Gui.qml` - -list=" -qtfrontend.PathOK -qtfrontend.PathEmptyPath -qtfrontend.PathWrongPath -qtfrontend.PathNotADir -qtfrontend.PathWrongPermissions -qtfrontend.PathDirEmpty - -errors.ErrUnknownError -errors.ErrEventAPILogout -errors.ErrUpgradeAPI -errors.ErrUpgradeJSON -errors.ErrUserAuth -errors.ErrQApplication -errors.ErrEmailExportFailed -errors.ErrEmailExportMissing -errors.ErrNothingToImport -errors.ErrEmailImportFailed -errors.ErrDraftImportFailed -errors.ErrDraftLabelFailed -errors.ErrEncryptMessageAttachment -errors.ErrEncryptMessage -errors.ErrNoInternetWhileImport -errors.ErrUnlockUser -errors.ErrSourceMessageNotSelected - -source.ErrCannotParseMail -source.ErrWrongLoginOrPassword -source.ErrWrongServerPathOrPort -source.ErrWrongAuthMethod -source.ErrIMAPFetchFailed - -qtfrontend.ErrLocalSourceLoadFailed -qtfrontend.ErrPMLoadFailed -qtfrontend.ErrRemoteSourceLoadFailed -qtfrontend.ErrLoadAccountList -qtfrontend.ErrExit -qtfrontend.ErrRetry -qtfrontend.ErrAsk -qtfrontend.ErrImportFailed -qtfrontend.ErrCreateLabelFailed -qtfrontend.ErrCreateFolderFailed -qtfrontend.ErrUpdateLabelFailed -qtfrontend.ErrUpdateFolderFailed -qtfrontend.ErrFillFolderName -qtfrontend.ErrSelectFolderColor -qtfrontend.ErrNoInternet - -qtfrontend.FolderTypeSystem -qtfrontend.FolderTypeLabel -qtfrontend.FolderTypeFolder -qtfrontend.FolderTypeExternal - -backend.ProgressInit -backend.ProgressLooping -backend.ErrPMAPIMessageTooLarge - -qtfrontend.StatusNoInternet -qtfrontend.StatusCheckingInternet -qtfrontend.StatusNewVersionAvailable -qtfrontend.StatusUpToDate -qtfrontend.StatusForceUpgrade -" - -first=true - - -if true; then - echo '// +build ignore' - echo '' - echo 'package main' - echo '' - echo 'import (' - echo ' "github.com/ProtonMail/Import-Export/backend"' - echo ' "github.com/ProtonMail/Import-Export/backend/source"' - echo ' "github.com/ProtonMail/Import-Export/backend/errors"' - echo ' "github.com/ProtonMail/Import-Export/frontend"' - echo ' "fmt"' - echo ')' - echo '' - echo 'func main(){' - echo ' checkValues := map[int]string{}' - echo ' checkDuplicates := map[string]bool{}' - echo ' fmt.Print("{")' - for c in $list - do - if ! $first; then - echo 'fmt.Print(",")' - fi - - if [[ $c =~ .*Err ]]; then - ## Add check that all Err have different value - echo 'if enumName,ok := checkValues[int('$c')]; ok {' - echo ' panic("Enum '$c' and "+enumName+" has same value")' - echo '}' - echo 'checkValues[int('$c')]="'$c'"' - fi - - cname=`echo $c | cut -d. -f2` - lowCase=${cname,} - - ## Add check that all qml enums have different value - echo 'if checkDuplicates["'$lowCase'"]{' - echo ' panic("Enum with same lowcase name as '$c' has already been registered")' - echo '}' - echo 'checkDuplicates["'$lowCase'"]=true' - - ## add value in lowercase - echo 'fmt.Printf("\"'$lowCase'\":%#v",'$c')' - - first=false - done - echo ' fmt.Print("}")' - echo '}' -fi > main.go - - -if true; then -echo -n "property var enums : JSON.parse('" -go run main.go || exit 5 -echo -n "')" -fi > out.qml - -rm main.go -sed -i "s/property var enums : JSON.parse.*$/`cat out.qml`/" ./qml/Gui.qml -rm out.qml - diff --git a/utils/release_notes.sh b/utils/release_notes.sh index 93234b17..1722e4b3 100755 --- a/utils/release_notes.sh +++ b/utils/release_notes.sh @@ -23,12 +23,6 @@ INFILE=$1 OUTFILE=${INFILE//.md/.html} -# Load props -APP_NAME="Import-Export app" -if [[ "$INFILE" =~ bridge ]]; then - APP_NAME="Bridge" -fi - CHANNEL=early if [[ "$INFILE" =~ stable ]]; then CHANNEL=stable @@ -36,9 +30,9 @@ fi # Check dependencies if ! which pandoc; then - echo "PANDOC NOT FOUND!\nPlease install pandoc in order to build release notes." + printf "PANDOC NOT FOUND!\nPlease install pandoc in order to build release notes." exit 1 fi # Build release notes -pandoc $INFILE -f markdown -t html -s -o $OUTFILE -c utils/release_notes.css --self-contained --section-divs --metadata title="Release notes - ProtonMail $APP_NAME - $CHANNEL" +pandoc "$INFILE" -f markdown -t html -s -o "$OUTFILE" -c utils/release_notes.css --self-contained --section-divs --metadata title="Release notes - ProtonMail Bridge - $CHANNEL"