ProtonMail
/
proton-bridge
mirror of https://github.com/ProtonMail/proton-bridge
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

GODT-1177: remove Import-Export from repo

This commit is contained in:
James Houlahan 2021-05-28 09:56:52 +02:00 committed by Jakub
parent 649195cc2b
commit ffb18adfd0
141 changed files with 140 additions and 8781 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.gitlab-ci.yml
View File

@ -168,26 +168,6 @@ build-darwin-qa:
paths:
- bridge_*.tgz
build-ie-darwin:
extends: .build-darwin-base
script:
- make build-ie
artifacts:
name: "ie-darwin-$CI_COMMIT_SHORT_SHA"
paths:
- ie_*.tgz
build-ie-darwin-qa:
extends: .build-darwin-base
only:
- web
script:
- BUILD_TAGS="build_qa" make build-ie
artifacts:
name: "ie-darwin-qa-$CI_COMMIT_SHORT_SHA"
paths:
- ie_*.tgz
# Stage: MIRROR
mirror-repo:

62
Makefile
View File

@ -7,29 +7,16 @@ TARGET_CMD?=Desktop-Bridge
TARGET_OS?=${GOOS}
## Build
.PHONY: build build-ie build-nogui build-ie-nogui build-launcher build-launcher-ie versioner hasher
.PHONY: build build-nogui build-launcher versioner hasher
# Keep version hardcoded so app build works also without Git repository.
BRIDGE_APP_VERSION?=1.8.12+git
IE_APP_VERSION?=1.3.3+git
APP_VERSION:=${BRIDGE_APP_VERSION}
SRC_ICO:=logo.ico
SRC_ICNS:=Bridge.icns
SRC_SVG:=logo.svg
TGT_ICNS:=Bridge.icns
EXE_NAME:=proton-bridge
CONFIGNAME:=bridge
WINDRES_DEFINE:=BUILD_BRIDGE
ifeq "${TARGET_CMD}" "Import-Export"
APP_VERSION:=${IE_APP_VERSION}
SRC_ICO:=ie.ico
SRC_ICNS:=ie.icns
SRC_SVG:=ie.svg
TGT_ICNS:=ImportExport.icns
EXE_NAME:=proton-ie
CONFIGNAME:=importExport
WINDRES_DEFINE:=BUILD_IE
endif
REVISION:=$(shell git rev-parse --short=10 HEAD)
BUILD_TIME:=$(shell date +%FT%T%z)
@ -41,7 +28,6 @@ ifneq "${BUILD_LDFLAGS}" ""
GO_LDFLAGS+=${BUILD_LDFLAGS}
endif
GO_LDFLAGS_LAUNCHER:=${GO_LDFLAGS}
GO_LDFLAGS_LAUNCHER+=$(addprefix -X main.,ConfigName=${CONFIGNAME} ExeName=proton-${APP})
ifeq "${TARGET_OS}" "windows"
GO_LDFLAGS_LAUNCHER+=-H=windowsgui
endif
@ -70,9 +56,6 @@ EXE_TARGET:=${DEPLOY_DIR}/${TARGET_OS}/${EXE}
EXE_QT_TARGET:=${DEPLOY_DIR}/${TARGET_OS}/${EXE_QT}
TGZ_TARGET:=bridge_${TARGET_OS}_${REVISION}.tgz
ifeq "${TARGET_CMD}" "Import-Export"
TGZ_TARGET:=ie_${TARGET_OS}_${REVISION}.tgz
endif
ifdef QT_API
VENDOR_TARGET:=prepare-vendor update-qt-docs
@ -82,15 +65,9 @@ endif
build: ${TGZ_TARGET}
build-ie:
TARGET_CMD=Import-Export $(MAKE) build
build-nogui: gofiles
go build ${BUILD_FLAGS} -o ${EXE_NAME} cmd/${TARGET_CMD}/main.go
build-ie-nogui:
TARGET_CMD=Import-Export $(MAKE) build-nogui
ifeq "${GOOS}" "windows"
PRERESOURCECMD:=cp ./resource.syso ./cmd/launcher/resource.syso
POSTRESOURCECMD:=rm -f ./cmd/launcher/resource.syso
@ -100,9 +77,6 @@ build-launcher: ${RESOURCE_FILE}
go build ${BUILD_FLAGS_LAUNCHER} -o launcher-${EXE} ./cmd/launcher/
${POSTRESOURCECMD}
build-launcher-ie:
TARGET_CMD=Import-Export $(MAKE) build-launcher
versioner:
go build ${BUILD_FLAGS} -o versioner utils/versioner/main.go
@ -124,7 +98,7 @@ ${DEPLOY_DIR}/darwin: ${EXE_TARGET}
mv ${EXE_TARGET}/Contents/MacOS/{${DIRNAME},${EXE_NAME}}; \
perl -i -pe"s/>${DIRNAME}/>${EXE_NAME}/g" ${EXE_TARGET}/Contents/Info.plist; \
fi
cp ./internal/frontend/share/icons/${SRC_ICNS} ${DARWINAPP_CONTENTS}/Resources/${TGT_ICNS}
cp ./internal/frontend/share/icons/${SRC_ICNS} ${DARWINAPP_CONTENTS}/Resources/${SRC_ICNS}
cp LICENSE ${DARWINAPP_CONTENTS}/Resources/
rm -rf "${DARWINAPP_CONTENTS}/Frameworks/QtWebEngine.framework"
rm -rf "${DARWINAPP_CONTENTS}/Frameworks/QtWebView.framework"
@ -155,7 +129,7 @@ WINDRES_YEAR:=$(shell date +%Y)
APP_VERSION_COMMA:=$(shell echo "${APP_VERSION}" | sed -e 's/[^0-9,.]*//g' -e 's/\./,/g')
resource.syso: ./internal/frontend/share/info.rc ./internal/frontend/share/icons/${SRC_ICO} .FORCE
rm -f ./*.syso
windres --target=pe-x86-64 -I ./internal/frontend/share/icons/ -D ${WINDRES_DEFINE} -D ICO_FILE=${SRC_ICO} -D EXE_NAME="${EXE_NAME}" -D FILE_VERSION="${APP_VERSION}" -D ORIGINAL_FILE_NAME="${EXE}" -D PRODUCT_VERSION="${APP_VERSION}" -D FILE_VERSION_COMMA=${APP_VERSION_COMMA} -D YEAR=${WINDRES_YEAR} -o $@ $<
windres --target=pe-x86-64 -I ./internal/frontend/share/icons/ -D ICO_FILE=${SRC_ICO} -D EXE_NAME="${EXE_NAME}" -D FILE_VERSION="${APP_VERSION}" -D ORIGINAL_FILE_NAME="${EXE}" -D PRODUCT_VERSION="${APP_VERSION}" -D FILE_VERSION_COMMA=${APP_VERSION_COMMA} -D YEAR=${WINDRES_YEAR} -o $@ $<
## Rules for therecipe/qt
.PHONY: prepare-vendor update-vendor update-qt-docs
@ -232,13 +206,11 @@ test: gofiles
./internal/events/... \
./internal/frontend/cli/... \
./internal/imap/... \
./internal/importexport/... \
./internal/locations/... \
./internal/logging/... \
./internal/metrics/... \
./internal/smtp/... \
./internal/store/... \
./internal/transfer/... \
./internal/updater/... \
./internal/users/... \
./internal/versioner/... \
@ -258,7 +230,6 @@ integration-test-bridge:
mocks:
mockgen --package mocks github.com/ProtonMail/proton-bridge/internal/users Locator,PanicHandler,CredentialsStorer,StoreMaker > internal/users/mocks/mocks.go
mockgen --package mocks github.com/ProtonMail/proton-bridge/pkg/listener Listener > internal/users/mocks/listener_mocks.go
mockgen --package mocks github.com/ProtonMail/proton-bridge/internal/transfer PanicHandler,IMAPClientProvider > internal/transfer/mocks/mocks.go
mockgen --package mocks github.com/ProtonMail/proton-bridge/internal/store PanicHandler,BridgeUser,ChangeNotifier > internal/store/mocks/mocks.go
mockgen --package mocks github.com/ProtonMail/proton-bridge/pkg/listener Listener > internal/store/mocks/utils_mocks.go
mockgen --package mocks github.com/ProtonMail/proton-bridge/pkg/pmapi Client,Manager > pkg/pmapi/mocks/mocks.go
@ -284,7 +255,7 @@ updates: install-go-mod-outdated
doc:
godoc -http=:6060
release-notes: release-notes/bridge_stable.html release-notes/bridge_early.html release-notes/ie_stable.html release-notes/ie_early.html
release-notes: release-notes/bridge_stable.html release-notes/bridge_early.html
release-notes/%.html: release-notes/%.md
./utils/release_notes.sh $^
@ -292,21 +263,17 @@ release-notes/%.html: release-notes/%.md
.PHONY: gofiles
# Following files are for the whole app so it makes sense to have them in bridge package.
# (Options like cmd or internal were considered and bridge package is the best place for them.)
gofiles: ./internal/bridge/credits.go ./internal/importexport/credits.go
gofiles: ./internal/bridge/credits.go
./internal/bridge/credits.go: ./utils/credits.sh go.mod
cd ./utils/ && ./credits.sh bridge
./internal/importexport/credits.go: ./utils/credits.sh go.mod
cd ./utils/ && ./credits.sh importexport
## Run and debug
.PHONY: run run-qt run-qt-cli run-nogui run-nogui-cli run-debug run-qml-preview run-ie-qml-preview run-ie run-ie-qt run-ie-qt-cli run-ie-nogui run-ie-nogui-cli clean-vendor clean-frontend-qt clean-frontend-qt-ie clean-frontend-qt-common clean
.PHONY: run run-qt run-qt-cli run-nogui run-nogui-cli run-debug run-qml-preview clean-vendor clean-frontend-qt clean-frontend-qt-common clean
LOG?=debug
LOG_IMAP?=client # client/server/all, or empty to turn it off
LOG_SMTP?=--log-smtp # empty to turn it off
RUN_FLAGS?=-m -l=${LOG} --log-imap=${LOG_IMAP} ${LOG_SMTP}
RUN_FLAGS_IE?=-m -l=${LOG}
run: run-nogui-cli
@ -325,24 +292,13 @@ run-debug:
run-qml-preview:
$(MAKE) -C internal/frontend/qt -f Makefile.local qmlpreview
run-ie-qml-preview:
$(MAKE) -C internal/frontend/qt-ie -f Makefile.local qmlpreview
run-ie:
TARGET_CMD=Import-Export RUN_FLAGS="${RUN_FLAGS_IE}" $(MAKE) run
run-ie-qt:
TARGET_CMD=Import-Export RUN_FLAGS="${RUN_FLAGS_IE}" $(MAKE) run-qt
run-ie-nogui:
TARGET_CMD=Import-Export RUN_FLAGS="${RUN_FLAGS_IE}" $(MAKE) run-nogui
clean-frontend-qt:
$(MAKE) -C internal/frontend/qt -f Makefile.local clean
clean-frontend-qt-ie:
$(MAKE) -C internal/frontend/qt-ie -f Makefile.local clean
# TODO: $(MAKE) -C internal/frontend/qt -f Makefile.local clean
clean-frontend-qt-common:
$(MAKE) -C internal/frontend/qt-common -f Makefile.local clean
# TODO: $(MAKE) -C internal/frontend/qt-common -f Makefile.local clean
clean-vendor: clean-frontend-qt clean-frontend-qt-ie clean-frontend-qt-common
clean-vendor: clean-frontend-qt clean-frontend-qt-common
rm -rf ./vendor
clean: clean-vendor

3
README.md
View File

@ -37,6 +37,8 @@ check the results.
More details [on the public website](https://protonmail.com/import-export).
The Import-Export app is developed in separate branch `master-ie`.
## Launchers
Launchers are binaries used to run the ProtonMail Bridge or Import-Export apps.
@ -69,7 +71,6 @@ or
### Integration testing
- `TEST_ENV`: set which env to use (fake or live)
- `TEST_APP`: set which app to test (bridge or ie)
- `TEST_ACCOUNTS`: set JSON file with configured accounts
- `TAGS`: set build tags for tests
- `FEATURES`: set feature dir, file or scenario to test

17
cmd/launcher/main.go
View File

@ -38,11 +38,10 @@ import (
"github.com/sirupsen/logrus"
)
const appName = "ProtonMail Launcher"
var (
ConfigName = "" // nolint[gochecknoglobals]
ExeName = "" // nolint[gochecknoglobals]
const (
appName = "ProtonMail Launcher"
configName = "bridge"
exeName = "proton-bridge"
)
func main() { // nolint[funlen]
@ -51,12 +50,12 @@ func main() { // nolint[funlen]
crashHandler := crash.NewHandler(reporter.ReportException)
defer crashHandler.HandlePanic()
locationsProvider, err := locations.NewDefaultProvider(filepath.Join(constants.VendorName, ConfigName))
locationsProvider, err := locations.NewDefaultProvider(filepath.Join(constants.VendorName, configName))
if err != nil {
logrus.WithError(err).Fatal("Failed to get locations provider")
}
locations := locations.New(locationsProvider, ConfigName)
locations := locations.New(locationsProvider, configName)
logsPath, err := locations.ProvideLogsPath()
if err != nil {
@ -87,9 +86,9 @@ func main() { // nolint[funlen]
versioner := versioner.New(updatesPath)
exe, err := getPathToUpdatedExecutable(ExeName, versioner, kr, reporter)
exe, err := getPathToUpdatedExecutable(exeName, versioner, kr, reporter)
if err != nil {
if exe, err = getFallbackExecutable(ExeName, versioner); err != nil {
if exe, err = getFallbackExecutable(exeName, versioner); err != nil {
logrus.WithError(err).Fatal("Failed to find any launchable executable")
}
}

11
dist/proton-ie.desktop vendored
View File

@ -1,11 +0,0 @@
[Desktop Entry]
Type=Application
Version=1.1
Name=ProtonMail Import-Export app
GenericName=ProtonMail Import-Export app for Linux
Comment=The Import-Export app helps you to migrate your emails from local files or remote IMAP servers to ProtonMail or simply export emails to local folder.
Icon=protonmail-import-export-app
Exec=protonmail-import-export-app
Terminal=false
Categories=Office;Email;Network
StartupWMClass=protonmail-import-export-app

135
doc/importexport.md
View File

@ -1,135 +0,0 @@
# Import-Export app
## Main blocks
This is basic overview of the main Import-Export blocks.
```mermaid
graph LR
S[ProtonMail server]
U[User]
subgraph "Import-Export app"
Users
Frontend["Qt / CLI"]
ImportExport
Transfer
Frontend --> ImportExport
Frontend --> Transfer
ImportExport --> Users
ImportExport --> Transfer
end
EML --> Transfer
MBOX --> Transfer
IMAP --> Transfer
S --> Transfer
Transfer --> EML
Transfer --> MBOX
Transfer --> S
U --> Frontend
```
## Code structure
More detailed graph of main types used in Import-Export app and connection between them.
```mermaid
graph TD
PM[ProtonMail Server]
EML[EML]
MBOX[MBOX]
IMAP[IMAP]
subgraph "Import-Export app"
subgraph "pkg users"
subgraph "pkg credentials"
CredStore[Store]
Creds[Credentials]
CredStore --> Creds
end
US[Users]
U[User]
US --> U
end
subgraph "pkg frontend"
CLI
Qt
end
subgraph "pkg importExport"
IE[ImportExport]
end
subgraph "pkg transfer"
Transfer
Rules
Progress
Provider
LocalProvider
EMLProvider
MBOXProvider
IMAPProvider
PMAPIProvider
Mailbox
Message
Transfer --> |source|Provider
Transfer --> |target|Provider
Transfer --> Rules
Transfer --> Progress
Provider --> LocalProvider
Provider --> EMLProvider
Provider --> MBOXProvider
Provider --> IMAPProvider
Provider --> PMAPIProvider
LocalProvider --> EMLProvider
LocalProvider --> MBOXProvider
Provider --> Mailbox
Provider --> Message
end
subgraph PMAPI
APIM[ClientManager]
APIC[Client]
APIM --> APIC
end
end
CLI --> IE
CLI --> Transfer
CLI --> Progress
Qt --> IE
Qt --> Transfer
Qt --> Progress
U --> CredStore
U --> Creds
US --> APIM
U --> APIM
PMAPIProvider --> APIM
EMLProvider --> EML
MBOXProvider --> MBOX
IMAPProvider --> IMAP
IE --> US
IE --> Transfer
APIC --> PM
```

7
doc/index.md
View File

@ -1,14 +1,9 @@
# Documentation
# Bridge Documentation
Documentation pages in order to read for a novice:
## Bridge
* [Bridge code](bridge.md)
* [Internal Bridge database](database.md)
* [Communication between Bridge, Client and Server](communication.md)
* [Encryption](encryption.md)
## Import-Export app
* [Import-Export code](importexport.md)

11
internal/bridge/bridge.go
View File

@ -68,8 +68,15 @@ func New(
clientManager.AllowProxy()
}
storeFactory := newStoreFactory(cache, sentryReporter, panicHandler, eventListener)
u := users.New(locations, panicHandler, eventListener, clientManager, credStorer, storeFactory, true)
u := users.New(
locations,
panicHandler,
eventListener,
clientManager,
credStorer,
newStoreFactory(cache, sentryReporter, panicHandler, eventListener),
)
b := &Bridge{
Users: u,

36
internal/frontend/share/info.rc Normal file
View File

@ -0,0 +1,36 @@
#define STRINGIZE_(x) #x
#define STRINGIZE(x) STRINGIZE_(x)
IDI_ICON1 ICON DISCARDABLE STRINGIZE(ICO_FILE)
#define FILE_COMMENTS "The Bridge is an application that runs on your computer in the background and seamlessly encrypts and decrypts your mail as it enters and leaves your computer."
#define FILE_DESCRIPTION "ProtonMail Bridge"
#define INTERNAL_NAME STRINGIZE(EXE_NAME)
#define PRODUCT_NAME "ProtonMail Bridge for Windows"
#define LEGAL_COPYRIGHT "(C) " STRINGIZE(YEAR) " Proton Technologies AG"
1 VERSIONINFO
FILEVERSION FILE_VERSION_COMMA,0
PRODUCTVERSION FILE_VERSION_COMMA,0
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "Comments", FILE_COMMENTS
VALUE "CompanyName", "Proton Technologies AG"
VALUE "FileDescription", FILE_DESCRIPTION
VALUE "FileVersion", STRINGIZE(FILE_VERSION)
VALUE "InternalName", INTERNAL_NAME
VALUE "LegalCopyright", LEGAL_COPYRIGHT
VALUE "OriginalFilename", STRINGIZE(ORIGINAL_FILE_NAME)
VALUE "ProductName", PRODUCT_NAME
VALUE "ProductVersion", STRINGIZE(PRODUCT_VERSION)
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x0409, 0x04B0
END
END

187
internal/importexport/importexport.go
View File

@ -1,187 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
// Package importexport provides core functionality of Import-Export app.
package importexport
import (
"bytes"
"context"
"github.com/ProtonMail/proton-bridge/internal/events"
"github.com/ProtonMail/proton-bridge/internal/transfer"
"github.com/ProtonMail/proton-bridge/internal/users"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/ProtonMail/proton-bridge/pkg/listener"
logrus "github.com/sirupsen/logrus"
)
var (
log = logrus.WithField("pkg", "importexport") //nolint[gochecknoglobals]
)
type ImportExport struct {
*users.Users
locations Locator
cache Cacher
panicHandler users.PanicHandler
eventListener listener.Listener
clientManager pmapi.Manager
}
func New(
locations Locator,
cache Cacher,
panicHandler users.PanicHandler,
eventListener listener.Listener,
clientManager pmapi.Manager,
credStorer users.CredentialsStorer,
) *ImportExport {
u := users.New(locations, panicHandler, eventListener, clientManager, credStorer, &storeFactory{}, false)
return &ImportExport{
Users: u,
locations: locations,
cache: cache,
panicHandler: panicHandler,
eventListener: eventListener,
clientManager: clientManager,
}
}
// ReportBug reports a new bug from the user.
func (ie *ImportExport) ReportBug(osType, osVersion, description, accountName, address, emailClient string) error {
return ie.clientManager.ReportBug(context.Background(), pmapi.ReportBugReq{
OS: osType,
OSVersion: osVersion,
Browser: emailClient,
Title: "[Import-Export] Bug",
Description: description,
Username: accountName,
Email: address,
})
}
// ReportFile submits import report file.
func (ie *ImportExport) ReportFile(osType, osVersion, accountName, address string, logdata []byte) error {
report := pmapi.ReportBugReq{
OS: osType,
OSVersion: osVersion,
Description: "An Import-Export report from the user swam down the river.",
Title: "[Import-Export] report file",
Username: accountName,
Email: address,
}
report.AddAttachment("log", "report.log", bytes.NewReader(logdata))
return ie.clientManager.ReportBug(context.Background(), report)
}
// GetLocalImporter returns transferrer from local EML or MBOX structure to ProtonMail account.
func (ie *ImportExport) GetLocalImporter(username, address, path string) (*transfer.Transfer, error) {
source := transfer.NewLocalProvider(path)
target, err := ie.getPMAPIProvider(username, address)
if err != nil {
return nil, err
}
logsPath, err := ie.locations.ProvideLogsPath()
if err != nil {
return nil, err
}
return transfer.New(ie.panicHandler, newImportMetricsManager(ie), logsPath, ie.cache.GetTransferDir(), source, target)
}
// GetRemoteImporter returns transferrer from remote IMAP to ProtonMail account.
func (ie *ImportExport) GetRemoteImporter(username, address, remoteUsername, remotePassword, host, port string) (*transfer.Transfer, error) {
source, err := transfer.NewIMAPProvider(remoteUsername, remotePassword, host, port)
if err != nil {
return nil, err
}
target, err := ie.getPMAPIProvider(username, address)
if err != nil {
return nil, err
}
logsPath, err := ie.locations.ProvideLogsPath()
if err != nil {
return nil, err
}
return transfer.New(ie.panicHandler, newImportMetricsManager(ie), logsPath, ie.cache.GetTransferDir(), source, target)
}
// GetEMLExporter returns transferrer from ProtonMail account to local EML structure.
func (ie *ImportExport) GetEMLExporter(username, address, path string) (*transfer.Transfer, error) {
source, err := ie.getPMAPIProvider(username, address)
if err != nil {
return nil, err
}
target := transfer.NewEMLProvider(path)
logsPath, err := ie.locations.ProvideLogsPath()
if err != nil {
return nil, err
}
return transfer.New(ie.panicHandler, newExportMetricsManager(ie), logsPath, ie.cache.GetTransferDir(), source, target)
}
// GetMBOXExporter returns transferrer from ProtonMail account to local MBOX structure.
func (ie *ImportExport) GetMBOXExporter(username, address, path string) (*transfer.Transfer, error) {
source, err := ie.getPMAPIProvider(username, address)
if err != nil {
return nil, err
}
target := transfer.NewMBOXProvider(path)
logsPath, err := ie.locations.ProvideLogsPath()
if err != nil {
return nil, err
}
return transfer.New(ie.panicHandler, newExportMetricsManager(ie), logsPath, ie.cache.GetTransferDir(), source, target)
}
func (ie *ImportExport) getPMAPIProvider(username, address string) (*transfer.PMAPIProvider, error) {
user, err := ie.Users.GetUser(username)
if err != nil {
return nil, err
}
addressID, err := user.GetAddressID(address)
if err != nil {
log.WithError(err).Info("Address does not exist, using all addresses")
}
provider, err := transfer.NewPMAPIProvider(user.GetClient(), user.ID(), addressID)
if err != nil {
return nil, err
}
go func() {
internetOffCh := ie.eventListener.ProvideChannel(events.InternetOffEvent)
internetOnCh := ie.eventListener.ProvideChannel(events.InternetOnEvent)
for {
select {
case <-internetOffCh:
provider.SetConnectionDown()
case <-internetOnCh:
provider.SetConnectionUp()
}
}
}()
return provider, nil
}

75
internal/importexport/metrics.go
View File

@ -1,75 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package importexport
import (
"strconv"
"github.com/ProtonMail/proton-bridge/internal/metrics"
"github.com/sirupsen/logrus"
)
type metricsManager struct {
ie *ImportExport
category metrics.Category
}
func newImportMetricsManager(ie *ImportExport) *metricsManager {
return &metricsManager{
ie: ie,
category: metrics.Import,
}
}
func newExportMetricsManager(ie *ImportExport) *metricsManager {
return &metricsManager{
ie: ie,
category: metrics.Export,
}
}
func (m *metricsManager) Load(numberOfMailboxes int) {
label := strconv.Itoa(numberOfMailboxes)
if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferLoad, metrics.Label(label))); err != nil {
logrus.WithError(err).Error("Failed to send metric")
}
}
func (m *metricsManager) Start() {
if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferStart, metrics.NoLabel)); err != nil {
logrus.WithError(err).Error("Failed to send metric")
}
}
func (m *metricsManager) Complete() {
if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferComplete, metrics.NoLabel)); err != nil {
logrus.WithError(err).Error("Failed to send metric")
}
}
func (m *metricsManager) Cancel() {
if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferCancel, metrics.NoLabel)); err != nil {
logrus.WithError(err).Error("Failed to send metric")
}
}
func (m *metricsManager) Fail() {
if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferFail, metrics.NoLabel)); err != nil {
logrus.WithError(err).Error("Failed to send metric")
}
}

35
internal/importexport/store_factory.go
View File

@ -1,35 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package importexport
import (
"github.com/ProtonMail/proton-bridge/internal/store"
)
// storeFactory implements dummy factory creating no store (not needed by Import-Export).
type storeFactory struct{}
// New does nothing.
func (f *storeFactory) New(user store.BridgeUser) (*store.Store, error) {
return nil, nil
}
// Remove does nothing.
func (f *storeFactory) Remove(userID string) error {
return nil
}

27
internal/importexport/types.go
View File

@ -1,27 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package importexport
type Locator interface {
ProvideLogsPath() (string, error)
Clear() error
}
type Cacher interface {
GetTransferDir() string
}

110
internal/transfer/mailbox.go
View File

@ -1,110 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"crypto/sha256"
"fmt"
"strings"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
)
var systemFolderMapping = map[string]string{ //nolint[gochecknoglobals]
"bin": "Trash",
"junk": "Spam",
"all": "All Mail",
"sent mail": "Sent",
"draft": "Drafts",
"important": "Starred",
// Add more translations.
}
// LeastUsedColor is intended to return color for creating a new inbox or label.
func LeastUsedColor(mailboxes []Mailbox) string {
usedColors := []string{}
for _, m := range mailboxes {
usedColors = append(usedColors, m.Color)
}
return pmapi.LeastUsedColor(usedColors)
}
// Mailbox is universal data holder of mailbox details for every provider.
type Mailbox struct {
ID string
Name string
Color string
IsExclusive bool
}
// IsSystemFolder returns true when ID corresponds to PM system folder.
func (m Mailbox) IsSystemFolder() bool {
return pmapi.IsSystemLabel(m.ID)
}
// Hash returns unique identifier to be used for matching.
func (m Mailbox) Hash() string {
return fmt.Sprintf("%x", sha256.Sum256([]byte(m.Name)))
}
// findMatchingMailboxes returns all matching mailboxes from `mailboxes`.
// Only one exclusive mailbox is included.
func (m Mailbox) findMatchingMailboxes(mailboxes []Mailbox) []Mailbox {
nameVariants := m.nameVariants()
isExclusiveIncluded := false
matches := []Mailbox{}
for i := range nameVariants {
nameVariant := nameVariants[len(nameVariants)-1-i]
for _, mailbox := range mailboxes {
if mailbox.IsExclusive && isExclusiveIncluded {
continue
}
if strings.ToLower(mailbox.Name) == nameVariant {
matches = append(matches, mailbox)
if mailbox.IsExclusive {
isExclusiveIncluded = true
}
}
}
}
return matches
}
// nameVariants returns all possible variants of the mailbox name.
// The best match (original name) is at the end of the slice.
// Variants are all in lower case. Examples:
// * Foo/bar -> [foo, bar, foo/bar]
// * x/Bin -> [x, trash, bin, x/bin]
// * a|b/c -> [a, b, c, a|b/c]
func (m Mailbox) nameVariants() (nameVariants []string) {
name := strings.ToLower(m.Name)
if strings.Contains(name, "/") || strings.Contains(name, "|") {
for _, slashPart := range strings.Split(name, "/") {
for _, part := range strings.Split(slashPart, "|") {
if mappedPart, ok := systemFolderMapping[part]; ok {
nameVariants = append(nameVariants, strings.ToLower(mappedPart))
}
nameVariants = append(nameVariants, part)
}
}
}
if mappedName, ok := systemFolderMapping[name]; ok {
nameVariants = append(nameVariants, strings.ToLower(mappedName))
}
return append(nameVariants, name)
}

111
internal/transfer/mailbox_test.go
View File

@ -1,111 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"testing"
r "github.com/stretchr/testify/require"
)
func TestLeastUsedColor(t *testing.T) {
var mailboxes []Mailbox
// Unset mailboxes, should use first available color
mailboxes = nil
r.Equal(t, "#7272a7", LeastUsedColor(mailboxes))
// No mailboxes at all, should use first available color
mailboxes = []Mailbox{}
r.Equal(t, "#7272a7", LeastUsedColor(mailboxes))
// All colors have same frequency, should use first available color
mailboxes = []Mailbox{
{Name: "Mbox1", Color: "#7272a7"},
{Name: "Mbox2", Color: "#cf5858"},
{Name: "Mbox3", Color: "#c26cc7"},
{Name: "Mbox4", Color: "#7569d1"},
{Name: "Mbox5", Color: "#69a9d1"},
{Name: "Mbox6", Color: "#5ec7b7"},
{Name: "Mbox7", Color: "#72bb75"},
{Name: "Mbox8", Color: "#c3d261"},
{Name: "Mbox9", Color: "#e6c04c"},
{Name: "Mbox10", Color: "#e6984c"},
{Name: "Mbox11", Color: "#8989ac"},
{Name: "Mbox12", Color: "#cf7e7e"},
{Name: "Mbox13", Color: "#c793ca"},
{Name: "Mbox14", Color: "#9b94d1"},
{Name: "Mbox15", Color: "#a8c4d5"},
{Name: "Mbox16", Color: "#97c9c1"},
{Name: "Mbox17", Color: "#9db99f"},
{Name: "Mbox18", Color: "#c6cd97"},
{Name: "Mbox19", Color: "#e7d292"},
{Name: "Mbox20", Color: "#dfb286"},
}
r.Equal(t, "#7272a7", LeastUsedColor(mailboxes))
// First three colors already used, but others wasn't. Should use first non-used one.
mailboxes = []Mailbox{
{Name: "Mbox1", Color: "#7272a7"},
{Name: "Mbox2", Color: "#cf5858"},
{Name: "Mbox3", Color: "#c26cc7"},
}
r.Equal(t, "#7569d1", LeastUsedColor(mailboxes))
}
func TestFindMatchingMailboxes(t *testing.T) {
mailboxes := []Mailbox{
{Name: "Inbox", IsExclusive: true},
{Name: "Sent", IsExclusive: true},
{Name: "Archive", IsExclusive: true},
{Name: "Foo", IsExclusive: false},
{Name: "hello/world", IsExclusive: true},
{Name: "Hello", IsExclusive: false},
{Name: "WORLD", IsExclusive: true},
{Name: "Trash", IsExclusive: true},
{Name: "Drafts", IsExclusive: true},
}
tests := []struct {
name string
wantNames []string
}{
{"inbox", []string{"Inbox"}},
{"foo", []string{"Foo"}},
{"hello", []string{"Hello"}},
{"world", []string{"WORLD"}},
{"hello/world", []string{"hello/world", "Hello"}},
{"hello|world", []string{"WORLD", "Hello"}},
{"nomailbox", []string{}},
{"bin", []string{"Trash"}},
{"root/bin", []string{"Trash"}},
{"draft", []string{"Drafts"}},
{"root/draft", []string{"Drafts"}},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
mailbox := Mailbox{Name: tc.name}
got := mailbox.findMatchingMailboxes(mailboxes)
gotNames := []string{}
for _, m := range got {
gotNames = append(gotNames, m.Name)
}
r.Equal(t, tc.wantNames, gotNames)
})
}
}

125
internal/transfer/message.go
View File

@ -1,125 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"mime"
"net/mail"
"time"
)
// Message is data holder passed between import and export.
type Message struct {
ID string
Unread bool
Body []byte
Sources []Mailbox
Targets []Mailbox
}
// sourceNames returns array of source mailbox names.
func (msg Message) sourceNames() (names []string) {
for _, mailbox := range msg.Sources {
names = append(names, mailbox.Name)
}
return
}
// targetNames returns array of target mailbox names.
func (msg Message) targetNames() (names []string) {
for _, mailbox := range msg.Targets {
names = append(names, mailbox.Name)
}
return
}
// MessageStatus holds status for message used by progress manager.
type MessageStatus struct {
eventTime time.Time // Time of adding message to the process.
sourceNames []string // Source mailbox names message is in.
SourceID string // Message ID at the source.
targetNames []string // Target mailbox names message is in.
targetID string // Message ID at the target (if any).
bodyHash string // Hash of the message body.
skipped bool
exported bool
imported bool
exportErr error
importErr error
// Info about message displayed to user.
// This is needed only for failed messages, but we cannot know in advance
// which message will fail. We could clear it once the message passed
// without any error. However, if we say one message takes about 100 bytes
// in average, it's about 100 MB per million of messages, which is fine.
Subject string
From string
Time time.Time
}
func (status *MessageStatus) String() string {
return fmt.Sprintf("%s (%s, %s, %s): %s", status.SourceID, status.Subject, status.From, status.Time, status.GetErrorMessage())
}
func (status *MessageStatus) setDetailsFromHeader(header mail.Header) {
dec := &mime.WordDecoder{}
status.Subject = header.Get("subject")
if subject, err := dec.Decode(status.Subject); err == nil {
status.Subject = subject
}
status.From = header.Get("from")
if from, err := dec.Decode(status.From); err == nil {
status.From = from
}
if msgTime, err := header.Date(); err == nil {
status.Time = msgTime
}
}
func (status *MessageStatus) hasError(includeMissing bool) bool {
return status.exportErr != nil || status.importErr != nil || (includeMissing && !status.skipped && !status.imported)
}
// GetErrorMessage returns error message.
func (status *MessageStatus) GetErrorMessage() string {
return status.getErrorMessage(true)
}
func (status *MessageStatus) getErrorMessage(includeMissing bool) string {
if status.skipped {
return ""
}
if status.exportErr != nil {
return fmt.Sprintf("failed to export: %s", status.exportErr)
}
if status.importErr != nil {
return fmt.Sprintf("failed to import: %s", status.importErr)
}
if includeMissing && !status.imported {
if !status.exported {
return "failed to import: lost before read"
}
return "failed to import: lost in the process"
}
return ""
}

397
internal/transfer/progress.go
View File

@ -1,397 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"crypto/sha256"
"fmt"
"sync"
"time"
"github.com/sirupsen/logrus"
)
// Progress maintains progress between import, export and user interface.
// Import and export update progress about processing messages and progress
// informs user interface, vice versa action (such as pause or resume) from
// user interface is passed down to import and export.
type Progress struct { //nolint[maligned]
log *logrus.Entry
lock sync.Locker
updateCh chan struct{}
messageCounted bool
messageCounts map[string]uint
messageStatuses map[string]*MessageStatus
pauseReason string
isStopped bool
fatalError error
fileReport *fileReport
}
func newProgress(log *logrus.Entry, fileReport *fileReport) Progress {
return Progress{
log: log,
lock: &sync.Mutex{},
updateCh: make(chan struct{}),
messageCounts: map[string]uint{},
messageStatuses: map[string]*MessageStatus{},
fileReport: fileReport,
}
}
// update is helper to notify listener for updates.
func (p *Progress) update() {
if p.updateCh == nil {
return
}
// In case no one listens for an update, do not block the whole progress.
go func() {
defer func() {
// updateCh can be closed at the end of progress which is fine.
if r := recover(); r != nil {
log.WithField("r", r).Warn("Failed to send update")
}
}()
select {
case p.updateCh <- struct{}{}:
case <-time.After(5 * time.Millisecond):
}
}()
}
// finish should be called as the last call once everything is done.
func (p *Progress) finish() {
p.lock.Lock()
defer p.lock.Unlock()
log.Debug("Progress finished")
p.cleanUpdateCh()
}
// fatal should be called once there is error with no possible continuation.
func (p *Progress) fatal(err error) {
p.lock.Lock()
defer p.lock.Unlock()
log.WithError(err).Error("Progress finished")
p.setStop()
p.fatalError = err
p.cleanUpdateCh()
}
func (p *Progress) cleanUpdateCh() {
if p.updateCh == nil {
return
}
close(p.updateCh)
p.updateCh = nil
}
func (p *Progress) countsFinal() {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
log.Info("Estimating count finished")
p.messageCounted = true
}
func (p *Progress) updateCount(mailbox string, count uint) {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
log.WithField("mailbox", mailbox).WithField("count", count).Debug("Mailbox count updated")
p.messageCounts[mailbox] = count
}
// addMessage should be called as soon as there is ID of the message.
func (p *Progress) addMessage(messageID string, sourceNames, targetNames []string) {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
p.log.WithField("id", messageID).Trace("Message added")
p.messageStatuses[messageID] = &MessageStatus{
eventTime: time.Now(),
sourceNames: sourceNames,
SourceID: messageID,
targetNames: targetNames,
}
}
// messageSkipped should be called once the message is skipped due to some
// filter such as time or folder and so on.
func (p *Progress) messageSkipped(messageID string) {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
p.log.WithField("id", messageID).Debug("Message skipped")
p.messageStatuses[messageID].skipped = true
p.logMessage(messageID)
}
// messageExported should be called right before message is exported.
func (p *Progress) messageExported(messageID string, body []byte, err error) {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
log := p.log.WithField("id", messageID)
if err != nil {
log = log.WithError(err)
}
log.Debug("Message exported")
status := p.messageStatuses[messageID]
status.exportErr = err
if err == nil {
status.exported = true
}
if len(body) > 0 {
status.bodyHash = fmt.Sprintf("%x", sha256.Sum256(body))
if header, err := getMessageHeader(body); err != nil {
log.WithError(err).Warning("Failed to parse headers for reporting")
} else {
status.setDetailsFromHeader(header)
}
}
// If export failed, no other step will be done with message and we can log it to the report file.
if err != nil {
p.logMessage(messageID)
}
}
// messageImported should be called right after message is imported.
func (p *Progress) messageImported(messageID, importID string, err error) {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
log := p.log.WithField("id", messageID)
if err != nil {
log = log.WithError(err)
}
log.Debug("Message imported")
p.messageStatuses[messageID].targetID = importID
p.messageStatuses[messageID].importErr = err
if err == nil {
p.messageStatuses[messageID].imported = true
}
// Import is the last step, now we can log the result to the report file.
p.logMessage(messageID)
}
// logMessage writes message status to log file.
func (p *Progress) logMessage(messageID string) {
if p.fileReport == nil {
return
}
p.fileReport.writeMessageStatus(p.messageStatuses[messageID])
}
// callWrap calls the callback and in case of problem it pause the process.
// Then it waits for user action to fix it and click on continue or abort.
// Every function doing I/O should be wrapped by this function to provide
// stopping and pausing functionality.
func (p *Progress) callWrap(callback func() error) {
for {
if p.shouldStop() {
break
}
err := callback()
if err == nil {
break
}
p.Pause("paused due to " + err.Error())
}
}
// shouldStop is utility for providers to automatically wait during pause
// and returned value determines whether the process shouls be fully stopped.
func (p *Progress) shouldStop() bool {
for p.IsPaused() {
time.Sleep(time.Second)
}
return p.IsStopped()
}
// GetUpdateChannel returns channel notifying any update from import or export.
func (p *Progress) GetUpdateChannel() chan struct{} {
p.lock.Lock()
defer p.lock.Unlock()
return p.updateCh
}
// Pause pauses the progress.
func (p *Progress) Pause(reason string) {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
p.log.Info("Progress paused")
p.pauseReason = reason
}
// Resume resumes the progress.
func (p *Progress) Resume() {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
p.log.Info("Progress resumed")
p.pauseReason = ""
}
// IsPaused returns whether progress is paused.
func (p *Progress) IsPaused() bool {
p.lock.Lock()
defer p.lock.Unlock()
return p.pauseReason != ""
}
// PauseReason returns pause reason.
func (p *Progress) PauseReason() string {
p.lock.Lock()
defer p.lock.Unlock()
return p.pauseReason
}
// Stop stops the process.
func (p *Progress) Stop() {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
p.log.Info("Progress stopped")
p.setStop()
// Once progress is stopped, some calls might be in progress. Results from
// those calls are irrelevant so we can close update channel sooner to not
// propagate any progress to user interface anymore.
p.cleanUpdateCh()
}
func (p *Progress) setStop() {
p.isStopped = true
p.pauseReason = "" // Clear pause to run paused code and stop it.
}
// IsStopped returns whether progress is stopped.
func (p *Progress) IsStopped() bool {
p.lock.Lock()
defer p.lock.Unlock()
return p.isStopped
}
// GetFatalError returns fatal error (progress failed and did not finish).
func (p *Progress) GetFatalError() error {
p.lock.Lock()
defer p.lock.Unlock()
return p.fatalError
}
// GetFailedMessages returns statuses of failed messages.
func (p *Progress) GetFailedMessages() []*MessageStatus {
p.lock.Lock()
defer p.lock.Unlock()
// Include lost messages in the process only when transfer is done.
includeMissing := p.updateCh == nil
statuses := []*MessageStatus{}
for _, status := range p.messageStatuses {
if status.hasError(includeMissing) {
statuses = append(statuses, status)
}
}
return statuses
}
// GetCounts returns counts of exported and imported messages.
func (p *Progress) GetCounts() ProgressCounts {
p.lock.Lock()
defer p.lock.Unlock()
counts := ProgressCounts{}
// Return counts only once total is estimated or the process already
// ended (for a case when it ended quickly to report it correctly).
if p.updateCh != nil && !p.messageCounted {
return counts
}
// Include lost messages in the process only when transfer is done.
includeMissing := p.updateCh == nil
for _, mailboxCount := range p.messageCounts {
counts.Total += mailboxCount
}
for _, status := range p.messageStatuses {
counts.Added++
if status.skipped {
counts.Skipped++
}
if status.exported {
counts.Exported++
}
if status.imported {
counts.Imported++
}
if status.hasError(includeMissing) {
counts.Failed++
}
}
return counts
}
// GenerateBugReport generates similar file to import log except private information.
func (p *Progress) GenerateBugReport() []byte {
bugReport := bugReport{}
for _, status := range p.messageStatuses {
bugReport.writeMessageStatus(status)
}
return bugReport.getData()
}
// FileReport returns path to generated defailed file report.
func (p *Progress) FileReport() string {
if p.fileReport == nil {
return ""
}
return p.fileReport.path
}

35
internal/transfer/progress_counts.go
View File

@ -1,35 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
// ProgressCounts holds counts counted by Progress.
type ProgressCounts struct {
Failed,
Skipped,
Imported,
Exported,
Added,
Total uint
}
// Progress returns ratio between processed messages (fully imported, skipped
// and failed ones) and total number of messages as percentage (0 - 1).
func (c *ProgressCounts) Progress() float32 {
progressed := c.Imported + c.Skipped + c.Failed
return float32(progressed) / float32(c.Total)
}

142
internal/transfer/progress_test.go
View File

@ -1,142 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"testing"
"time"
"github.com/pkg/errors"
a "github.com/stretchr/testify/assert"
r "github.com/stretchr/testify/require"
)
func TestProgressUpdateCount(t *testing.T) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
progress.updateCount("inbox", 10)
progress.updateCount("archive", 20)
progress.updateCount("inbox", 12)
progress.updateCount("sent", 5)
progress.updateCount("foo", 4)
progress.updateCount("foo", 5)
progress.finish()
counts := progress.GetCounts()
r.Equal(t, uint(42), counts.Total)
}
func TestProgressAddingMessages(t *testing.T) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
// msg1 has no problem.
progress.addMessage("msg1", []string{}, []string{})
progress.messageExported("msg1", []byte(""), nil)
progress.messageImported("msg1", "", nil)
// msg2 has an import problem.
progress.addMessage("msg2", []string{}, []string{})
progress.messageExported("msg2", []byte(""), nil)
progress.messageImported("msg2", "", errors.New("failed import"))
// msg3 has an export problem.
progress.addMessage("msg3", []string{}, []string{})
progress.messageExported("msg3", []byte(""), errors.New("failed export"))
// msg4 has an export problem and import is also called.
progress.addMessage("msg4", []string{}, []string{})
progress.messageExported("msg4", []byte(""), errors.New("failed export"))
progress.messageImported("msg4", "", nil)
// msg5 is skipped.
progress.addMessage("msg5", []string{}, []string{})
progress.messageSkipped("msg5")
progress.finish()
counts := progress.GetCounts()
a.Equal(t, uint(5), counts.Added)
a.Equal(t, uint(2), counts.Exported)
a.Equal(t, uint(2), counts.Imported)
a.Equal(t, uint(1), counts.Skipped)
a.Equal(t, uint(3), counts.Failed)
errorsMap := map[string]string{}
for _, status := range progress.GetFailedMessages() {
errorsMap[status.SourceID] = status.GetErrorMessage()
}
a.Equal(t, map[string]string{
"msg2": "failed to import: failed import",
"msg3": "failed to export: failed export",
"msg4": "failed to export: failed export",
}, errorsMap)
}
func TestProgressFinish(t *testing.T) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
progress.finish()
r.Nil(t, progress.updateCh)
r.NotPanics(t, func() { progress.addMessage("msg", []string{}, []string{}) })
}
func TestProgressFatalError(t *testing.T) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
progress.fatal(errors.New("fatal error"))
r.Nil(t, progress.updateCh)
r.NotPanics(t, func() { progress.addMessage("msg", []string{}, []string{}) })
}
func TestFailUnpauseAndStops(t *testing.T) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
progress.Pause("pausing")
progress.fatal(errors.New("fatal error"))
r.Nil(t, progress.updateCh)
r.True(t, progress.isStopped)
r.False(t, progress.IsPaused())
r.Eventually(t, progress.shouldStop, time.Second, 10*time.Millisecond)
}
func TestStopClosesUpdates(t *testing.T) {
progress := newProgress(log, nil)
ch := progress.updateCh
progress.Stop()
r.Nil(t, progress.updateCh)
r.PanicsWithError(t, "send on closed channel", func() { ch <- struct{}{} })
}
func drainProgressUpdateChannel(progress *Progress) {
// updateCh is not needed to drain under tests - timeout is implemented.
// But timeout takes time which would slow down tests.
go func() {
for range progress.updateCh {
}
}()
}

51
internal/transfer/provider.go
View File

@ -1,51 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
// Provider provides interface for common operation with provider.
type Provider interface {
// ID is used for generating transfer ID by combining source and target ID.
ID() string
// Mailboxes returns all available mailboxes.
// Provider used as source returns only non-empty maibloxes.
// Provider used as target does not return all mail maiblox.
Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error)
}
// SourceProvider provides interface of provider with support of export.
type SourceProvider interface {
Provider
// TransferTo exports messages based on rules to channel.
TransferTo(transferRules, *Progress, chan<- Message)
}
// TargetProvider provides interface of provider with support of import.
type TargetProvider interface {
Provider
// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
DefaultMailboxes(sourceMailbox Mailbox) (targetMailboxes []Mailbox)
// CreateMailbox creates new mailbox to be used as target in transfer rules.
CreateMailbox(Mailbox) (Mailbox, error)
// TransferFrom imports messages from channel.
TransferFrom(transferRules, *Progress, <-chan Message)
}

68
internal/transfer/provider_eml.go
View File

@ -1,68 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
// EMLProvider implements import and export to/from EML file structure.
type EMLProvider struct {
root string
}
// NewEMLProvider creates EMLProvider.
func NewEMLProvider(root string) *EMLProvider {
return &EMLProvider{
root: root,
}
}
// ID is used for generating transfer ID by combining source and target ID.
// We want to keep the same rules for import from or export to local files
// no matter exact path, therefore it returns constant. The same as EML.
func (p *EMLProvider) ID() string {
return "local" //nolint[goconst]
}
// Mailboxes returns all available folder names from root of EML files.
// In case the same folder name is used more than once (for example root/a/foo
// and root/b/foo), it's treated as the same folder.
func (p *EMLProvider) Mailboxes(includeEmpty, includeAllMail bool) (mailboxes []Mailbox, err error) {
// Special case for exporting--we don't know the path before setup if finished.
if p.root == "" {
return
}
var folderNames []string
if includeEmpty {
folderNames, err = getFolderNames(p.root)
} else {
folderNames, err = getFolderNamesWithFileSuffix(p.root, ".eml")
}
if err != nil {
return nil, err
}
for _, folderName := range folderNames {
mailboxes = append(mailboxes, Mailbox{
ID: "",
Name: folderName,
Color: "",
IsExclusive: false,
})
}
return mailboxes, nil
}

134
internal/transfer/provider_eml_source.go
View File

@ -1,134 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"io/ioutil"
"os"
"path/filepath"
"github.com/pkg/errors"
)
// TransferTo exports messages based on rules to channel.
func (p *EMLProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
log.Info("Started transfer from EML to channel")
defer log.Info("Finished transfer from EML to channel")
filePathsPerFolder, err := p.getFilePathsPerFolder(rules)
if err != nil {
progress.fatal(err)
return
}
if len(filePathsPerFolder) == 0 {
return
}
// This list is not filtered by time but instead going throgh each file
// twice or keeping all in memory we will tell rough estimation which
// will be updated during processing each file.
for folderName, filePaths := range filePathsPerFolder {
if progress.shouldStop() {
break
}
progress.updateCount(folderName, uint(len(filePaths)))
}
progress.countsFinal()
for folderName, filePaths := range filePathsPerFolder {
// No error guaranteed by getFilePathsPerFolder.
rule, _ := rules.getRuleBySourceMailboxName(folderName)
log.WithField("rule", rule).Debug("Processing rule")
p.exportMessages(rule, filePaths, progress, ch)
}
}
func (p *EMLProvider) getFilePathsPerFolder(rules transferRules) (map[string][]string, error) {
filePaths, err := getFilePathsWithSuffix(p.root, ".eml")
if err != nil {
return nil, err
}
filePathsMap := map[string][]string{}
for _, filePath := range filePaths {
folder := filepath.Base(filepath.Dir(filepath.Join(p.root, filePath)))
_, err := rules.getRuleBySourceMailboxName(folder)
if err != nil {
log.WithField("msg", filePath).Trace("Message skipped due to folder name")
continue
}
filePathsMap[folder] = append(filePathsMap[folder], filePath)
}
return filePathsMap, nil
}
func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *Progress, ch chan<- Message) {
for _, filePath := range filePaths {
if progress.shouldStop() {
break
}
msg, err := p.exportMessage(rule, filePath)
progress.addMessage(filePath, msg.sourceNames(), msg.targetNames())
// Read and check time in body only if the rule specifies it
// to not waste energy.
if err == nil && rule.HasTimeLimit() {
msgTime, msgTimeErr := getMessageTime(msg.Body)
if msgTimeErr != nil {
err = msgTimeErr
} else if !rule.isTimeInRange(msgTime) {
log.WithField("msg", filePath).Debug("Message skipped due to time")
progress.messageSkipped(filePath)
continue
}
}
progress.messageExported(filePath, msg.Body, err)
if err == nil {
ch <- msg
}
}
}
func (p *EMLProvider) exportMessage(rule *Rule, filePath string) (Message, error) {
fullFilePath := filepath.Clean(filepath.Join(p.root, filePath))
file, err := os.Open(fullFilePath) //nolint[gosec]
if err != nil {
return Message{}, errors.Wrap(err, "failed to open message")
}
defer file.Close() //nolint[errcheck]
body, err := ioutil.ReadAll(file)
if err != nil {
return Message{}, errors.Wrap(err, "failed to read message")
}
return Message{
ID: filePath,
Unread: false,
Body: body,
Sources: []Mailbox{rule.SourceMailbox},
Targets: rule.TargetMailboxes,
}, nil
}

88
internal/transfer/provider_eml_target.go
View File

@ -1,88 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"io/ioutil"
"os"
"path/filepath"
"github.com/hashicorp/go-multierror"
)
// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
func (p *EMLProvider) DefaultMailboxes(sourceMailbox Mailbox) []Mailbox {
return []Mailbox{{
Name: sourceMailbox.Name,
}}
}
// CreateMailbox does nothing. Folders are created dynamically during the import.
func (p *EMLProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) {
return mailbox, nil
}
// TransferFrom imports messages from channel.
func (p *EMLProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) {
log.Info("Started transfer from channel to EML")
defer log.Info("Finished transfer from channel to EML")
err := p.createFolders(rules)
if err != nil {
progress.fatal(err)
return
}
for msg := range ch {
for progress.shouldStop() {
break
}
err := p.writeFile(msg)
progress.messageImported(msg.ID, "", err)
}
}
func (p *EMLProvider) createFolders(rules transferRules) error {
for rule := range rules.iterateActiveRules() {
for _, mailbox := range rule.TargetMailboxes {
path := filepath.Join(p.root, sanitizeFileName(mailbox.Name))
if err := os.MkdirAll(path, os.ModePerm); err != nil {
return err
}
}
}
return nil
}
func (p *EMLProvider) writeFile(msg Message) error {
fileName := sanitizeFileName(filepath.Base(msg.ID))
if filepath.Ext(fileName) != ".eml" {
fileName += ".eml"
}
var err error
for _, mailbox := range msg.Targets {
path := filepath.Join(p.root, mailbox.Name, fileName)
if localErr := ioutil.WriteFile(path, msg.Body, 0600); localErr != nil {
err = multierror.Append(err, localErr)
}
}
return err
}

126
internal/transfer/provider_eml_test.go
View File

@ -1,126 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"io/ioutil"
"os"
"testing"
"time"
r "github.com/stretchr/testify/require"
)
func newTestEMLProvider(path string) *EMLProvider {
if path == "" {
path = "testdata/eml"
}
return NewEMLProvider(path)
}
func TestEMLProviderMailboxes(t *testing.T) {
provider := newTestEMLProvider("")
tests := []struct {
includeEmpty bool
wantMailboxes []Mailbox
}{
{true, []Mailbox{
{Name: "Foo"},
{Name: "Inbox"},
{Name: "eml"},
}},
{false, []Mailbox{
{Name: "Foo"},
{Name: "Inbox"},
}},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
mailboxes, err := provider.Mailboxes(tc.includeEmpty, false)
r.NoError(t, err)
r.Equal(t, tc.wantMailboxes, mailboxes)
})
}
}
func TestEMLProviderTransferTo(t *testing.T) {
provider := newTestEMLProvider("")
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupEMLRules(rules)
testTransferTo(t, rules, provider, []string{
"Foo/msg.eml",
"Inbox/msg.eml",
})
}
func TestEMLProviderTransferFrom(t *testing.T) {
dir, err := ioutil.TempDir("", "eml")
r.NoError(t, err)
defer os.RemoveAll(dir) //nolint[errcheck]
provider := newTestEMLProvider(dir)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupEMLRules(rules)
testTransferFrom(t, rules, provider, []Message{
{ID: "Foo/msg.eml", Body: getTestMsgBody("msg"), Targets: []Mailbox{{Name: "Foo"}}},
})
checkEMLFileStructure(t, dir, []string{
"Foo/msg.eml",
})
}
func TestEMLProviderTransferFromTo(t *testing.T) {
dir, err := ioutil.TempDir("", "eml")
r.NoError(t, err)
defer os.RemoveAll(dir) //nolint[errcheck]
source := newTestEMLProvider("")
target := newTestEMLProvider(dir)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupEMLRules(rules)
testTransferFromTo(t, rules, source, target, 5*time.Second)
checkEMLFileStructure(t, dir, []string{
"Foo/msg.eml",
"Inbox/msg.eml",
})
}
func setupEMLRules(rules transferRules) {
_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
}
func checkEMLFileStructure(t *testing.T, root string, expectedFiles []string) {
files, err := getFilePathsWithSuffix(root, ".eml")
r.NoError(t, err)
r.Equal(t, expectedFiles, files)
}

123
internal/transfer/provider_imap.go
View File

@ -1,123 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"net"
"strings"
"github.com/emersion/go-imap"
"github.com/emersion/go-sasl"
)
type IMAPClientProvider interface {
Capability() (map[string]bool, error)
Support(capability string) (bool, error)
State() imap.ConnState
SupportAuth(mech string) (bool, error)
Authenticate(auth sasl.Client) error
Login(username, password string) error
List(ref, name string, ch chan *imap.MailboxInfo) error
Select(name string, readOnly bool) (*imap.MailboxStatus, error)
Fetch(seqset *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) error
UidFetch(seqset *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) error
}
// IMAPProvider implements export from IMAP server.
type IMAPProvider struct {
username string
password string
addr string
clientDialer func(addr string) (IMAPClientProvider, error)
client IMAPClientProvider
timeIt *timeIt
}
// NewIMAPProvider returns new IMAPProvider.
func NewIMAPProvider(username, password, host, port string) (*IMAPProvider, error) {
return newIMAPProvider(imapClientDial, username, password, host, port)
}
func newIMAPProvider(clientDialer func(string) (IMAPClientProvider, error), username, password, host, port string) (*IMAPProvider, error) {
p := &IMAPProvider{
username: username,
password: password,
addr: net.JoinHostPort(host, port),
timeIt: newTimeIt("imap"),
clientDialer: clientDialer,
}
if err := p.auth(); err != nil {
return nil, err
}
return p, nil
}
// ID is used for generating transfer ID by combining source and target ID.
// We want to keep the same rules for import from any IMAP server, therefore
// it returns constant.
func (p *IMAPProvider) ID() string {
return "imap"
}
// Mailboxes returns all available folder names from root of EML files.
// In case the same folder name is used more than once (for example root/a/foo
// and root/b/foo), it's treated as the same folder.
func (p *IMAPProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
mailboxesInfo, err := p.list()
if err != nil {
return nil, err
}
mailboxes := []Mailbox{}
for _, mailbox := range mailboxesInfo {
hasNoSelect := false
for _, attrib := range mailbox.Attributes {
if strings.ToLower(attrib) == "\\noselect" {
hasNoSelect = true
break
}
}
if hasNoSelect {
continue
}
if !includeEmpty || true {
mailboxStatus, err := p.selectIn(mailbox.Name)
if err != nil {
return nil, err
}
if mailboxStatus.Messages == 0 {
continue
}
}
mailboxes = append(mailboxes, Mailbox{
ID: "",
Name: mailbox.Name,
Color: "",
IsExclusive: false,
})
}
return mailboxes, nil
}

66
internal/transfer/provider_imap_errors.go
View File

@ -1,66 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
// imapError is base for all IMAP errors.
type imapError struct {
Message string
Err error
}
func (e imapError) Error() string {
return e.Message + ": " + e.Err.Error()
}
func (e imapError) Unwrap() error {
return e.Err
}
func (e imapError) Cause() error {
return e.Err
}
// ErrIMAPConnection is error representing connection issues.
type ErrIMAPConnection struct {
imapError
}
func (e ErrIMAPConnection) Is(target error) bool {
_, ok := target.(*ErrIMAPConnection)
return ok
}
// ErrIMAPAuth is error representing authentication issues.
type ErrIMAPAuth struct {
imapError
}
func (e ErrIMAPAuth) Is(target error) bool {
_, ok := target.(*ErrIMAPAuth)
return ok
}
// ErrIMAPAuthMethod is error representing wrong auth method.
type ErrIMAPAuthMethod struct {
imapError
}
func (e ErrIMAPAuthMethod) Is(target error) bool {
_, ok := target.(*ErrIMAPAuthMethod)
return ok
}

250
internal/transfer/provider_imap_source.go
View File

@ -1,250 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"io/ioutil"
"github.com/emersion/go-imap"
)
type imapMessageInfo struct {
id string
uid uint32
size uint32
}
const (
imapPageSize = uint32(2000) // Optimized on Gmail.
imapMaxFetchSize = uint32(50 * 1000 * 1000) // Size in octets. If 0, it will use one fetch per message.
)
// TransferTo exports messages based on rules to channel.
func (p *IMAPProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
log.Info("Started transfer from IMAP to channel")
defer log.Info("Finished transfer from IMAP to channel")
p.timeIt.clear()
defer p.timeIt.logResults()
imapMessageInfoMap := p.loadMessageInfoMap(rules, progress)
for rule := range rules.iterateActiveRules() {
messagesInfo := imapMessageInfoMap[rule.SourceMailbox.Name]
if messagesInfo == nil {
log.WithField("rule", rule).Warn("Rule has no message info")
continue
}
log.WithField("rule", rule).Debug("Processing rule")
p.transferTo(rule, messagesInfo, progress, ch)
}
}
func (p *IMAPProvider) loadMessageInfoMap(rules transferRules, progress *Progress) map[string]map[string]imapMessageInfo {
res := map[string]map[string]imapMessageInfo{}
for rule := range rules.iterateActiveRules() {
if progress.shouldStop() {
break
}
mailboxName := rule.SourceMailbox.Name
var mailbox *imap.MailboxStatus
progress.callWrap(func() error {
var err error
mailbox, err = p.selectIn(mailboxName)
return err
})
if mailbox == nil {
log.WithField("rule", rule.SourceMailbox.Name).Warn("Failed to select into mailbox")
continue
}
if mailbox.Messages == 0 {
continue
}
messagesInfo := p.loadMessagesInfo(rule, progress, mailbox.UidValidity, mailbox.Messages)
res[rule.SourceMailbox.Name] = messagesInfo
progress.updateCount(rule.SourceMailbox.Name, uint(len(messagesInfo)))
}
progress.countsFinal()
return res
}
func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValidity, count uint32) map[string]imapMessageInfo {
p.timeIt.start("load", rule.SourceMailbox.Name)
defer p.timeIt.stop("load", rule.SourceMailbox.Name)
log := log.WithField("mailbox", rule.SourceMailbox.Name)
messagesInfo := map[string]imapMessageInfo{}
fetchItems := []imap.FetchItem{imap.FetchUid, imap.FetchRFC822Size}
if rule.HasTimeLimit() {
fetchItems = append(fetchItems, imap.FetchEnvelope)
}
processMessageCallback := func(imapMessage *imap.Message) {
if rule.HasTimeLimit() {
t := imapMessage.Envelope.Date.Unix()
if t != 0 && !rule.isTimeInRange(t) {
log.WithField("uid", imapMessage.Uid).Debug("Message skipped due to time")
return
}
}
id := getUniqueMessageID(rule.SourceMailbox.Name, uidValidity, imapMessage.Uid)
// We use ID as key to ensure we have every unique message only once.
// Some IMAP servers responded twice the same message...
messagesInfo[id] = imapMessageInfo{
id: id,
uid: imapMessage.Uid,
size: imapMessage.Size,
}
progress.addMessage(id, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames())
}
pageStart := uint32(1)
pageEnd := imapPageSize
for {
if progress.shouldStop() || pageStart > count {
break
}
// Some servers do not accept message sequence number higher than the total count.
if pageEnd > count {
pageEnd = count
}
seqSet := &imap.SeqSet{}
seqSet.AddRange(pageStart, pageEnd)
err := p.fetch(rule.SourceMailbox.Name, seqSet, fetchItems, processMessageCallback)
if err != nil {
log.WithError(err).WithField("idx", seqSet).Warning("Load batch fetch failed, trying one by one")
for ; pageStart <= pageEnd; pageStart++ {
seqSet := &imap.SeqSet{}
seqSet.AddNum(pageStart)
if err := p.fetch(rule.SourceMailbox.Name, seqSet, fetchItems, processMessageCallback); err != nil {
log.WithError(err).WithField("idx", seqSet).Warning("Load fetch failed, skipping the message")
}
}
}
pageStart = pageEnd + 1
pageEnd += imapPageSize
}
return messagesInfo
}
func (p *IMAPProvider) transferTo(rule *Rule, messagesInfo map[string]imapMessageInfo, progress *Progress, ch chan<- Message) {
progress.callWrap(func() error {
_, err := p.selectIn(rule.SourceMailbox.Name)
return err
})
seqSet := &imap.SeqSet{}
seqSetSize := uint32(0)
uidToID := map[uint32]string{}
for _, messageInfo := range messagesInfo {
if progress.shouldStop() {
break
}
if seqSetSize != 0 && (seqSetSize+messageInfo.size) > imapMaxFetchSize {
log.WithField("mailbox", rule.SourceMailbox.Name).WithField("seq", seqSet).WithField("size", seqSetSize).Debug("Fetching messages")
p.exportMessages(rule, progress, ch, seqSet, uidToID)
seqSet = &imap.SeqSet{}
seqSetSize = 0
uidToID = map[uint32]string{}
}
seqSet.AddNum(messageInfo.uid)
seqSetSize += messageInfo.size
uidToID[messageInfo.uid] = messageInfo.id
}
if len(uidToID) != 0 {
log.WithField("mailbox", rule.SourceMailbox.Name).WithField("seq", seqSet).WithField("size", seqSetSize).Debug("Fetching messages")
p.exportMessages(rule, progress, ch, seqSet, uidToID)
}
}
func (p *IMAPProvider) exportMessages(rule *Rule, progress *Progress, ch chan<- Message, seqSet *imap.SeqSet, uidToID map[uint32]string) {
section := &imap.BodySectionName{}
items := []imap.FetchItem{imap.FetchUid, imap.FetchFlags, section.FetchItem()}
processMessageCallback := func(imapMessage *imap.Message) {
if progress.shouldStop() {
return
}
id, ok := uidToID[imapMessage.Uid]
// Sometimes, server sends not requested messages.
if !ok {
log.WithField("uid", imapMessage.Uid).Warning("Message skipped: not requested")
return
}
// Sometimes, server sends message twice, once with body and once without it.
bodyReader := imapMessage.GetBody(section)
if bodyReader == nil {
log.WithField("uid", imapMessage.Uid).Warning("Message skipped: no body")
return
}
body, err := ioutil.ReadAll(bodyReader)
progress.messageExported(id, body, err)
if err == nil {
msg := p.exportMessage(rule, id, imapMessage, body)
p.timeIt.stop("fetch", rule.SourceMailbox.Name)
ch <- msg
p.timeIt.start("fetch", rule.SourceMailbox.Name)
}
}
p.timeIt.start("fetch", rule.SourceMailbox.Name)
progress.callWrap(func() error {
return p.uidFetch(rule.SourceMailbox.Name, seqSet, items, processMessageCallback)
})
p.timeIt.stop("fetch", rule.SourceMailbox.Name)
}
func (p *IMAPProvider) exportMessage(rule *Rule, id string, imapMessage *imap.Message, body []byte) Message {
unread := true
for _, flag := range imapMessage.Flags {
if flag == imap.SeenFlag {
unread = false
}
}
return Message{
ID: id,
Unread: unread,
Body: body,
Sources: []Mailbox{rule.SourceMailbox},
Targets: rule.TargetMailboxes,
}
}
func getUniqueMessageID(mailboxName string, uidValidity, uid uint32) string {
return fmt.Sprintf("%s_%d:%d", mailboxName, uidValidity, uid)
}

100
internal/transfer/provider_imap_test.go
View File

@ -1,100 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"testing"
"github.com/emersion/go-imap"
gomock "github.com/golang/mock/gomock"
"github.com/pkg/errors"
r "github.com/stretchr/testify/require"
)
func newTestIMAPProvider(t *testing.T, m mocks) *IMAPProvider {
m.imapClientProvider.EXPECT().State().Return(imap.ConnectedState).AnyTimes()
m.imapClientProvider.EXPECT().Capability().Return(map[string]bool{
"AUTH": true,
}, nil).AnyTimes()
dialer := func(string) (IMAPClientProvider, error) {
return m.imapClientProvider, nil
}
provider, err := newIMAPProvider(dialer, "user", "pass", "host", "42")
r.NoError(t, err)
return provider
}
func TestProviderIMAPLoadMessagesInfo(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
provider := newTestIMAPProvider(t, m)
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
rule := &Rule{SourceMailbox: Mailbox{Name: "Mailbox"}}
uidValidity := 1
count := 2200
failingIndex := 2100
m.imapClientProvider.EXPECT().Select(rule.SourceMailbox.Name, gomock.Any()).Return(&imap.MailboxStatus{}, nil).AnyTimes()
m.imapClientProvider.EXPECT().
Fetch(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(seqSet *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) error {
defer close(ch)
for _, seq := range seqSet.Set {
for i := seq.Start; i <= seq.Stop; i++ {
if int(i) == failingIndex {
return errors.New("internal server error")
}
ch <- &imap.Message{
SeqNum: i,
Uid: i * 10,
Size: i * 100,
}
}
}
return nil
}).
// 2200 messages is split into two batches (2000 and 200),
// the second one fails and makes 200 calls (one-by-one).
// Plus two failed requests are repeated `imapRetries` times.
Times(2 + 200 + (2 * (imapRetries - 1)))
messageInfo := provider.loadMessagesInfo(rule, &progress, uint32(uidValidity), uint32(count))
r.Equal(t, count-1, len(messageInfo)) // One message produces internal server error.
for index := 1; index <= count; index++ {
uid := index * 10
key := fmt.Sprintf("%s_%d:%d", rule.SourceMailbox.Name, uidValidity, uid)
if index == failingIndex {
r.Empty(t, messageInfo[key])
continue
}
r.Equal(t, imapMessageInfo{
id: key,
uid: uint32(uid),
size: uint32(index * 100),
}, messageInfo[key])
}
}

311
internal/transfer/provider_imap_utils.go
View File

@ -1,311 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"crypto/tls"
"fmt"
"net"
"net/http"
"strings"
"time"
imapID "github.com/ProtonMail/go-imap-id"
"github.com/ProtonMail/proton-bridge/internal/constants"
"github.com/emersion/go-imap"
imapClient "github.com/emersion/go-imap/client"
"github.com/emersion/go-sasl"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const (
imapDialTimeout = 5 * time.Second
imapRetries = 10
imapReconnectTimeout = 30 * time.Minute
imapReconnectSleep = time.Minute
protonStatusURL = "http://protonstatus.com/vpn_status"
)
type imapErrorLogger struct {
log *logrus.Entry
}
func (l *imapErrorLogger) Printf(f string, v ...interface{}) {
l.log.Errorf(f, v...)
}
func (l *imapErrorLogger) Println(v ...interface{}) {
l.log.Errorln(v...)
}
func imapClientDial(addr string) (IMAPClientProvider, error) {
if _, err := net.DialTimeout("tcp", addr, imapDialTimeout); err != nil {
return nil, errors.Wrap(err, "failed to dial server")
}
client, err := imapClientDialHelper(addr)
if err == nil {
client.ErrorLog = &imapErrorLogger{logrus.WithField("pkg", "imap-client")}
// Logrus `WriterLevel` fails for big messages because of bufio.MaxScanTokenSize limit.
// Also, this spams a lot, uncomment once needed during development.
// client.SetDebug(imap.NewDebugWriter(
// logrus.WithField("pkg", "imap/client").WriterLevel(logrus.TraceLevel),
// logrus.WithField("pkg", "imap/server").WriterLevel(logrus.TraceLevel),
// ))
}
return client, err
}
func imapClientDialHelper(addr string) (*imapClient.Client, error) {
host, _, _ := net.SplitHostPort(addr)
if host == "127.0.0.1" {
return imapClient.Dial(addr)
}
// IMAP mail.yahoo.com has problem with golang TLS 1.3 implementation
// with weird behaviour, i.e., Yahoo does not return error during dial
// or handshake but server does logs out right after successful login
// leaving no time to perform any action.
// Limiting TLS to version 1.2 is working just fine.
var tlsConf *tls.Config
if strings.Contains(strings.ToLower(host), "yahoo") {
log.Warning("Yahoo server detected: limiting maximal TLS version to 1.2.")
tlsConf = &tls.Config{MaxVersion: tls.VersionTLS12} //nolint[gosec] G402
}
return imapClient.DialTLS(addr, tlsConf)
}
func (p *IMAPProvider) ensureConnection(callback func() error) error {
return p.ensureConnectionAndSelection(callback, "")
}
func (p *IMAPProvider) ensureConnectionAndSelection(callback func() error, ensureSelectedIn string) error {
var callErr error
for i := 1; i <= imapRetries; i++ {
callErr = callback()
if callErr == nil {
return nil
}
log.WithField("attempt", i).WithError(callErr).Warning("IMAP call failed, trying reconnect")
err := p.tryReconnect(ensureSelectedIn)
if err != nil {
return err
}
}
return errors.Wrap(callErr, "too many retries")
}
func (p *IMAPProvider) tryReconnect(ensureSelectedIn string) error {
start := time.Now()
var previousErr error
for {
if time.Since(start) > imapReconnectTimeout {
return previousErr
}
err := checkConnection()
log.WithError(err).Debug("Connection check")
if err != nil {
time.Sleep(imapReconnectSleep)
previousErr = err
continue
}
err = p.reauth()
log.WithError(err).Debug("Reauth")
if err != nil {
time.Sleep(imapReconnectSleep)
previousErr = err
continue
}
if ensureSelectedIn != "" {
_, err = p.client.Select(ensureSelectedIn, true)
log.WithError(err).Debug("Reselect")
if err != nil {
previousErr = err
continue
}
}
break
}
return nil
}
func (p *IMAPProvider) reauth() error {
var state imap.ConnState
// In some cases once go-imap fails, we cannot issue another command
// because it would dead-lock. Let's simply ignore it, we want to open
// new connection anyway.
ch := make(chan struct{})
go func() {
defer close(ch)
if _, err := p.client.Capability(); err != nil {
state = p.client.State()
}
}()
select {
case <-ch:
case <-time.After(30 * time.Second):
}
log.WithField("addr", p.addr).WithField("state", state).Debug("Reconnecting")
p.client = nil
return p.auth()
}
func (p *IMAPProvider) auth() error { //nolint[funlen]
log := log.WithField("addr", p.addr)
log.Info("Connecting to server")
client, err := p.clientDialer(p.addr)
if err != nil {
return ErrIMAPConnection{imapError{Err: err, Message: "failed to connect to server"}}
}
p.client = client
log.Info("Connected")
if (p.client.State() & imap.AuthenticatedState) == imap.AuthenticatedState {
return nil
}
capability, err := p.client.Capability()
log.WithField("capability", capability).WithError(err).Debug("Server capability")
if err != nil {
return ErrIMAPConnection{imapError{Err: err, Message: "failed to get capabilities"}}
}
// SASL AUTH PLAIN
if ok, _ := p.client.SupportAuth("PLAIN"); p.client.State() == imap.NotAuthenticatedState && ok {
log.Debug("Trying plain auth")
authPlain := sasl.NewPlainClient("", p.username, p.password)
if err = p.client.Authenticate(authPlain); err != nil {
return ErrIMAPAuth{imapError{Err: err, Message: "plain auth failed"}}
}
}
// LOGIN: if the server reports the IMAP4rev1 capability then it is standards conformant and must support login.
if ok, _ := p.client.Support("IMAP4rev1"); p.client.State() == imap.NotAuthenticatedState && ok {
log.Debug("Trying login")
if err = p.client.Login(p.username, p.password); err != nil {
return ErrIMAPAuth{imapError{Err: err, Message: "login failed"}}
}
}
if p.client.State() == imap.NotAuthenticatedState {
return ErrIMAPAuthMethod{imapError{Err: err, Message: "unknown auth method"}}
}
log.Info("Logged in")
if c, ok := p.client.(*imapClient.Client); ok {
idClient := imapID.NewClient(c)
if ok, err := idClient.SupportID(); err == nil && ok {
serverID, err := idClient.ID(imapID.ID{
imapID.FieldName: "ImportExport",
imapID.FieldVersion: constants.Version,
})
log.WithField("ID", serverID).WithError(err).Debug("Server info")
}
}
return err
}
func (p *IMAPProvider) list() (mailboxes []*imap.MailboxInfo, err error) {
err = p.ensureConnection(func() error {
mailboxesCh := make(chan *imap.MailboxInfo)
doneCh := make(chan error)
go func() {
doneCh <- p.client.List("", "*", mailboxesCh)
}()
for mailbox := range mailboxesCh {
mailboxes = append(mailboxes, mailbox)
}
return <-doneCh
})
return
}
func (p *IMAPProvider) selectIn(mailboxName string) (mailbox *imap.MailboxStatus, err error) {
err = p.ensureConnection(func() error {
mailbox, err = p.client.Select(mailboxName, true)
return err
})
return
}
func (p *IMAPProvider) fetch(ensureSelectedIn string, seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error {
return p.fetchHelper(false, ensureSelectedIn, seqSet, items, processMessageCallback)
}
func (p *IMAPProvider) uidFetch(ensureSelectedIn string, seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error {
return p.fetchHelper(true, ensureSelectedIn, seqSet, items, processMessageCallback)
}
func (p *IMAPProvider) fetchHelper(uid bool, ensureSelectedIn string, seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error {
return p.ensureConnectionAndSelection(func() error {
messagesCh := make(chan *imap.Message)
doneCh := make(chan error)
go func() {
if uid {
doneCh <- p.client.UidFetch(seqSet, items, messagesCh)
} else {
doneCh <- p.client.Fetch(seqSet, items, messagesCh)
}
}()
for message := range messagesCh {
processMessageCallback(message)
}
err := <-doneCh
return err
}, ensureSelectedIn)
}
// checkConnection returns an error if there is no internet connection.
// Note we don't want to use client manager because it only reports connection
// issues with API; we are only interested here whether we can reach
// third-party IMAP servers.
func checkConnection() error {
client := &http.Client{Timeout: time.Second * 10}
resp, err := client.Get(protonStatusURL)
if err != nil {
return err
}
_ = resp.Body.Close()
if resp.StatusCode != 200 {
return fmt.Errorf("HTTP status code %d", resp.StatusCode)
}
return nil
}

68
internal/transfer/provider_local.go
View File

@ -1,68 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
// LocalProvider implements import from local EML and MBOX file structure.
type LocalProvider struct {
root string
emlProvider *EMLProvider
mboxProvider *MBOXProvider
}
func NewLocalProvider(root string) *LocalProvider {
return &LocalProvider{
root: root,
emlProvider: NewEMLProvider(root),
mboxProvider: NewMBOXProvider(root),
}
}
// ID is used for generating transfer ID by combining source and target ID.
// We want to keep the same rules for import from or export to local files
// no matter exact path, therefore it returns constant.
// The same as EML and MBOX.
func (p *LocalProvider) ID() string {
return "local" //nolint[goconst]
}
// Mailboxes returns all available folder names from root of EML and MBOX files.
func (p *LocalProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
mailboxes, err := p.emlProvider.Mailboxes(includeEmpty, includeAllMail)
if err != nil {
return nil, err
}
mboxMailboxes, err := p.mboxProvider.Mailboxes(includeEmpty, includeAllMail)
if err != nil {
return nil, err
}
for _, mboxMailbox := range mboxMailboxes {
found := false
for _, mailboxes := range mailboxes {
if mboxMailbox.Name == mailboxes.Name {
found = true
break
}
}
if !found {
mailboxes = append(mailboxes, mboxMailbox)
}
}
return mailboxes, nil
}

42
internal/transfer/provider_local_source.go
View File

@ -1,42 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"sync"
)
// TransferTo exports messages based on rules to channel.
func (p *LocalProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
log.Info("Started transfer from EML and MBOX to channel")
defer log.Info("Finished transfer from EML and MBOX to channel")
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
p.emlProvider.TransferTo(rules, progress, ch)
}()
go func() {
defer wg.Done()
p.mboxProvider.TransferTo(rules, progress, ch)
}()
wg.Wait()
}

77
internal/transfer/provider_local_test.go
View File

@ -1,77 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"testing"
r "github.com/stretchr/testify/require"
)
func newTestLocalProvider(path string) *LocalProvider {
if path == "" {
path = "testdata/emlmbox"
}
return NewLocalProvider(path)
}
func TestLocalProviderMailboxes(t *testing.T) {
provider := newTestLocalProvider("")
tests := []struct {
includeEmpty bool
wantMailboxes []Mailbox
}{
{true, []Mailbox{
{Name: "Foo"},
{Name: "emlmbox"},
{Name: "Inbox"},
}},
{false, []Mailbox{
{Name: "Foo"},
{Name: "Inbox"},
}},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
mailboxes, err := provider.Mailboxes(tc.includeEmpty, false)
r.NoError(t, err)
r.Equal(t, tc.wantMailboxes, mailboxes)
})
}
}
func TestLocalProviderTransferTo(t *testing.T) {
provider := newTestLocalProvider("")
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupEMLMBOXRules(rules)
testTransferTo(t, rules, provider, []string{
"Foo/msg.eml",
"Inbox.mbox:1",
})
}
func setupEMLMBOXRules(rules transferRules) {
_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
}

102
internal/transfer/provider_mbox.go
View File

@ -1,102 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
)
// MBOXProvider implements import and export to/from MBOX structure.
type MBOXProvider struct {
root string
}
func NewMBOXProvider(root string) *MBOXProvider {
return &MBOXProvider{
root: root,
}
}
// ID is used for generating transfer ID by combining source and target ID.
// We want to keep the same rules for import from or export to local files
// no matter exact path, therefore it returns constant. The same as EML.
func (p *MBOXProvider) ID() string {
return "local" //nolint[goconst]
}
// Mailboxes returns all available folder names from root of EML files.
// In case the same folder name is used more than once (for example root/a/foo
// and root/b/foo), it's treated as the same folder.
func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
filePaths, err := getAllPathsWithSuffix(p.root, ".mbox")
if err != nil {
return nil, err
}
mailboxNames := map[string]bool{}
for _, filePath := range filePaths {
fileName := filepath.Base(filePath)
filePath, err := p.handleAppleMailMBOXStructure(filePath)
if err != nil {
log.WithError(err).Warn("Failed to handle MBOX structure")
continue
}
mailboxName := strings.TrimSuffix(fileName, ".mbox")
mailboxNames[mailboxName] = true
labels, err := getGmailLabelsFromMboxFile(filepath.Join(p.root, filePath))
if err != nil {
log.WithError(err).Error("Failed to get gmail labels from mbox file")
continue
}
for label := range labels {
mailboxNames[label] = true
}
}
mailboxes := []Mailbox{}
for mailboxName := range mailboxNames {
mailboxes = append(mailboxes, Mailbox{
ID: "",
Name: mailboxName,
Color: "",
IsExclusive: false,
})
}
return mailboxes, nil
}
// handleAppleMailMBOXStructure changes the path of mailbox directory to
// the path of mbox file. Apple Mail MBOX exports has this structure:
// `Folder.mbox` directory with `mbox` file inside.
// Example: `Folder.mbox/mbox` (and this function converts `Folder.mbox`
// to `Folder.mbox/mbox`).
func (p *MBOXProvider) handleAppleMailMBOXStructure(filePath string) (string, error) {
if info, err := os.Stat(filepath.Join(p.root, filePath)); err == nil && info.IsDir() {
if _, err := os.Stat(filepath.Join(p.root, filePath, "mbox")); err != nil {
return "", errors.Wrap(err, "wrong mbox structure")
}
return filepath.Join(filePath, "mbox"), nil
}
return filePath, nil
}

118
internal/transfer/provider_mbox_gmail_labels.go
View File

@ -1,118 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"bufio"
"bytes"
"io"
"mime"
"os"
"strings"
)
type stringSet map[string]bool
const xGmailLabelsHeader = "X-Gmail-Labels"
// filteredOutGmailLabels is set of labels which we don't want to show to users
// as they might be auto-generated by Gmail and unwanted.
var filteredOutGmailLabels = []string{ //nolint[gochecknoglobals]
"Unread",
"Opened",
"IMAP_Junk",
"IMAP_NonJunk",
"IMAP_NotJunk",
"IMAP_$NotJunk",
}
func getGmailLabelsFromMboxFile(filePath string) (stringSet, error) {
f, err := os.Open(filePath) //nolint[gosec]
if err != nil {
return nil, err
}
return getGmailLabelsFromMboxReader(f)
}
func getGmailLabelsFromMboxReader(f io.Reader) (stringSet, error) {
allLabels := stringSet{}
// Scanner is not used as it does not support long lines and some mbox
// files contain very long lines even though that should not be happening.
r := bufio.NewReader(f)
for {
b, isPrefix, err := r.ReadLine()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
for isPrefix {
_, isPrefix, err = r.ReadLine()
if err != nil {
break
}
}
if bytes.HasPrefix(b, []byte(xGmailLabelsHeader)) {
for label := range getGmailLabelsFromValue(string(b)) {
allLabels[label] = true
}
}
}
return allLabels, nil
}
func getGmailLabelsFromMessage(body []byte) (stringSet, error) {
header, err := getMessageHeader(body)
if err != nil {
return nil, err
}
labels := header.Get(xGmailLabelsHeader)
return getGmailLabelsFromValue(labels), nil
}
func getGmailLabelsFromValue(value string) stringSet {
value = strings.TrimPrefix(value, xGmailLabelsHeader+":")
if decoded, err := new(mime.WordDecoder).DecodeHeader(value); err != nil {
log.WithError(err).Error("Failed to decode header")
} else {
value = decoded
}
labels := stringSet{}
for _, label := range strings.Split(value, ",") {
label = strings.TrimSpace(label)
if label == "" {
continue
}
skip := false
for _, filteredOutLabel := range filteredOutGmailLabels {
if label == filteredOutLabel {
skip = true
break
}
}
if skip {
continue
}
labels[label] = true
}
return labels
}

135
internal/transfer/provider_mbox_gmail_labels_test.go
View File

@ -1,135 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"strings"
"testing"
r "github.com/stretchr/testify/require"
)
func TestGetGmailLabelsFromMboxReader(t *testing.T) {
mboxFile := `From - Mon May 4 16:40:31 2020
Subject: Test 1
X-Gmail-Labels: Foo,Bar
hello
From - Mon May 4 16:40:31 2020
Subject: Test 2
X-Gmail-Labels: Foo , Baz
hello
From - Mon May 4 16:40:31 2020
Subject: Test 3
X-Gmail-Labels: ,
hello
From - Mon May 4 16:40:31 2020
Subject: Test 4
X-Gmail-Labels:
hello
From - Mon May 4 16:40:31 2020
Subject: Test 5
hello
`
mboxReader := strings.NewReader(mboxFile)
labels, err := getGmailLabelsFromMboxReader(mboxReader)
r.NoError(t, err)
r.Equal(t, toSet("Foo", "Bar", "Baz"), labels)
}
func TestGetGmailLabelsFromMessage(t *testing.T) {
tests := []struct {
body string
wantLabels stringSet
}{
{`Subject: One
X-Gmail-Labels: Foo,Bar
Hello
`, toSet("Foo", "Bar")},
{`Subject: Two
X-Gmail-Labels: Foo , Bar ,
Hello
`, toSet("Foo", "Bar")},
{`Subject: Three
X-Gmail-Labels: ,
Hello
`, toSet()},
{`Subject: Four
X-Gmail-Labels:
Hello
`, toSet()},
{`Subject: Five
Hello
`, toSet()},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.body), func(t *testing.T) {
labels, err := getGmailLabelsFromMessage([]byte(tc.body))
r.NoError(t, err)
r.Equal(t, tc.wantLabels, labels)
})
}
}
func TestGetGmailLabelsFromValue(t *testing.T) {
tests := []struct {
value string
wantLabels stringSet
}{
{"Foo,Bar", toSet("Foo", "Bar")},
{" Foo , Bar ", toSet("Foo", "Bar")},
{" Foo , Bar , ", toSet("Foo", "Bar")},
{" Foo Bar ", toSet("Foo Bar")},
{" , ", toSet()},
{" ", toSet()},
{"", toSet()},
{"=?UTF-8?Q?Archived,Category_personal,test_=F0=9F=98=80=F0=9F=99=83?=", toSet("Archived", "Category personal", "test 😀🙃")},
{"IMAP_NotJunk,Foo,Opened,bar,Unread", toSet("Foo", "bar")},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.value), func(t *testing.T) {
labels := getGmailLabelsFromValue(tc.value)
r.Equal(t, tc.wantLabels, labels)
})
}
}
func toSet(items ...string) stringSet {
set := map[string]bool{}
for _, item := range items {
set[item] = true
}
return set
}

255
internal/transfer/provider_mbox_source.go
View File

@ -1,255 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/emersion/go-mbox"
"github.com/pkg/errors"
)
// TransferTo exports messages based on rules to channel.
func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
log.Info("Started transfer from MBOX to channel")
defer log.Info("Finished transfer from MBOX to channel")
filePathsPerFolder, err := p.getFilePathsPerFolder()
if err != nil {
progress.fatal(err)
return
}
if len(filePathsPerFolder) == 0 {
return
}
for folderName, filePaths := range filePathsPerFolder {
log.WithField("folder", folderName).Debug("Estimating folder counts")
for _, filePath := range filePaths {
if progress.shouldStop() {
break
}
p.updateCount(progress, filePath)
}
}
progress.countsFinal()
for folderName, filePaths := range filePathsPerFolder {
log.WithField("folder", folderName).Debug("Processing folder")
for _, filePath := range filePaths {
if progress.shouldStop() {
break
}
p.transferTo(rules, progress, ch, folderName, filePath)
}
}
}
func (p *MBOXProvider) getFilePathsPerFolder() (map[string][]string, error) {
filePaths, err := getAllPathsWithSuffix(p.root, ".mbox")
if err != nil {
return nil, err
}
filePathsMap := map[string][]string{}
for _, filePath := range filePaths {
fileName := filepath.Base(filePath)
filePath, err := p.handleAppleMailMBOXStructure(filePath)
// Skip unsupported MBOX structures. It was already filtered out in configuration step.
if err != nil {
continue
}
folder := strings.TrimSuffix(fileName, ".mbox")
filePathsMap[folder] = append(filePathsMap[folder], filePath)
}
return filePathsMap, nil
}
func (p *MBOXProvider) updateCount(progress *Progress, filePath string) {
mboxReader := p.openMbox(progress, filePath)
if mboxReader == nil {
return
}
count := 0
for {
_, err := mboxReader.NextMessage()
if err == io.EOF {
break
} else if err != nil {
progress.fatal(err)
break
}
count++
}
progress.updateCount(filePath, uint(count))
}
func (p *MBOXProvider) transferTo(rules transferRules, progress *Progress, ch chan<- Message, folderName, filePath string) {
mboxReader := p.openMbox(progress, filePath)
if mboxReader == nil {
return
}
index := 0
for {
if progress.shouldStop() {
break
}
index++
id := fmt.Sprintf("%s:%d", filePath, index)
msgReader, err := mboxReader.NextMessage()
if err == io.EOF {
break
} else if err != nil {
progress.fatal(err)
break
}
msg, err := p.exportMessage(rules, folderName, id, msgReader)
progress.addMessage(id, msg.sourceNames(), msg.targetNames())
if err == nil && len(msg.Targets) == 0 {
progress.messageSkipped(id)
continue
}
progress.messageExported(id, msg.Body, err)
if err == nil {
ch <- msg
}
}
}
func (p *MBOXProvider) exportMessage(rules transferRules, folderName, id string, msgReader io.Reader) (Message, error) {
body, err := ioutil.ReadAll(msgReader)
if err != nil {
return Message{}, errors.Wrap(err, "failed to read message")
}
msgRules := p.getMessageRules(rules, folderName, id, body)
sources := p.getMessageSources(msgRules)
targets := p.getMessageTargets(msgRules, id, body)
return Message{
ID: id,
Unread: false,
Body: body,
Sources: sources,
Targets: targets,
}, nil
}
func (p *MBOXProvider) getMessageRules(rules transferRules, folderName, id string, body []byte) []*Rule {
msgRules := []*Rule{}
folderRule, err := rules.getRuleBySourceMailboxName(folderName)
if err != nil {
log.WithField("msg", id).WithField("source", folderName).Debug("Message source doesn't have a rule")
} else if folderRule.Active {
msgRules = append(msgRules, folderRule)
}
gmailLabels, err := getGmailLabelsFromMessage(body)
if err != nil {
log.WithError(err).Error("Failed to get gmail labels, ")
} else {
for label := range gmailLabels {
rule, err := rules.getRuleBySourceMailboxName(label)
if err != nil {
log.WithField("msg", id).WithField("source", label).Debug("Message source doesn't have a rule")
continue
}
if rule.Active {
msgRules = append(msgRules, rule)
}
}
}
return msgRules
}
func (p *MBOXProvider) getMessageSources(msgRules []*Rule) []Mailbox {
sources := []Mailbox{}
for _, rule := range msgRules {
sources = append(sources, rule.SourceMailbox)
}
return sources
}
func (p *MBOXProvider) getMessageTargets(msgRules []*Rule, id string, body []byte) []Mailbox {
targets := []Mailbox{}
haveExclusiveMailbox := false
for _, rule := range msgRules {
// Read and check time in body only if the rule specifies it
// to not waste energy.
if rule.HasTimeLimit() {
msgTime, err := getMessageTime(body)
if err != nil {
log.WithError(err).Error("Failed to parse time, time check skipped")
} else if !rule.isTimeInRange(msgTime) {
log.WithField("msg", id).WithField("source", rule.SourceMailbox.Name).Debug("Message skipped due to time")
continue
}
}
for _, newTarget := range rule.TargetMailboxes {
// msgRules is sorted. The first rule is based on the folder name,
// followed by the order from X-Gmail-Labels. The rule based on
// the folder name should have priority for exclusive target.
if newTarget.IsExclusive && haveExclusiveMailbox {
continue
}
found := false
for _, target := range targets {
if target.Hash() == newTarget.Hash() {
found = true
break
}
}
if found {
continue
}
if newTarget.IsExclusive {
haveExclusiveMailbox = true
}
targets = append(targets, newTarget)
}
}
return targets
}
func (p *MBOXProvider) openMbox(progress *Progress, mboxPath string) *mbox.Reader {
mboxPath = filepath.Join(p.root, mboxPath)
mboxFile, err := os.Open(mboxPath) //nolint[gosec]
if os.IsNotExist(err) {
return nil
} else if err != nil {
progress.fatal(err)
return nil
}
return mbox.NewReader(mboxFile)
}

97
internal/transfer/provider_mbox_target.go
View File

@ -1,97 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"os"
"path/filepath"
"strings"
"time"
"github.com/emersion/go-mbox"
"github.com/hashicorp/go-multierror"
)
// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
func (p *MBOXProvider) DefaultMailboxes(sourceMailbox Mailbox) []Mailbox {
return []Mailbox{{
Name: sourceMailbox.Name,
}}
}
// CreateMailbox does nothing. Files are created dynamically during the import.
func (p *MBOXProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) {
return mailbox, nil
}
// TransferFrom imports messages from channel.
func (p *MBOXProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) {
log.Info("Started transfer from channel to MBOX")
defer log.Info("Finished transfer from channel to MBOX")
for msg := range ch {
if progress.shouldStop() {
break
}
err := p.writeMessage(msg)
progress.messageImported(msg.ID, "", err)
}
}
func (p *MBOXProvider) writeMessage(msg Message) error {
var multiErr error
for _, mailbox := range msg.Targets {
mboxName := sanitizeFileName(mailbox.Name)
if !strings.HasSuffix(mboxName, ".mbox") {
mboxName += ".mbox"
}
mboxPath := filepath.Join(p.root, mboxName)
mboxFile, err := os.OpenFile(filepath.Clean(mboxPath), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
multiErr = multierror.Append(multiErr, err)
continue
}
msgFrom := ""
msgTime := time.Now()
if header, err := getMessageHeader(msg.Body); err == nil {
if date, err := header.Date(); err == nil {
msgTime = date
}
if addresses, err := header.AddressList("from"); err == nil && len(addresses) > 0 {
msgFrom = addresses[0].Address
}
}
mboxWriter := mbox.NewWriter(mboxFile)
messageWriter, err := mboxWriter.CreateMessage(msgFrom, msgTime)
if err != nil {
multiErr = multierror.Append(multiErr, err)
continue
}
_, err = messageWriter.Write(msg.Body)
if err != nil {
multiErr = multierror.Append(multiErr, err)
continue
}
}
return multiErr
}

223
internal/transfer/provider_mbox_test.go
View File

@ -1,223 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"io/ioutil"
"os"
"testing"
"time"
r "github.com/stretchr/testify/require"
)
func newTestMBOXProvider(path string) *MBOXProvider {
if path == "" {
path = "testdata/mbox"
}
return NewMBOXProvider(path)
}
func TestMBOXProviderMailboxes(t *testing.T) {
tests := []struct {
provider *MBOXProvider
includeEmpty bool
wantMailboxes []Mailbox
}{
{newTestMBOXProvider(""), true, []Mailbox{
{Name: "All Mail"},
{Name: "Foo"},
{Name: "Bar"},
{Name: "Inbox"},
}},
{newTestMBOXProvider(""), false, []Mailbox{
{Name: "All Mail"},
{Name: "Foo"},
{Name: "Bar"},
{Name: "Inbox"},
}},
{newTestMBOXProvider("testdata/mbox-applemail"), true, []Mailbox{
{Name: "All Mail"},
{Name: "Foo"},
{Name: "Bar"},
}},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
mailboxes, err := tc.provider.Mailboxes(tc.includeEmpty, false)
r.NoError(t, err)
r.ElementsMatch(t, tc.wantMailboxes, mailboxes)
})
}
}
func TestMBOXProviderTransferTo(t *testing.T) {
provider := newTestMBOXProvider("")
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupMBOXRules(rules)
msgs := testTransferTo(t, rules, provider, []string{
"All Mail.mbox:1",
"All Mail.mbox:2",
"Foo.mbox:1",
"Inbox.mbox:1",
})
got := map[string][]string{}
for _, msg := range msgs {
got[msg.ID] = msg.targetNames()
}
r.Equal(t, map[string][]string{
"All Mail.mbox:1": {"Archive", "Foo"}, // Bar is not in rules.
"All Mail.mbox:2": {"Archive", "Foo"},
"Foo.mbox:1": {"Foo"},
"Inbox.mbox:1": {"Inbox"},
}, got)
}
func TestMBOXProviderTransferToAppleMail(t *testing.T) {
provider := newTestMBOXProvider("testdata/mbox-applemail")
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupMBOXRules(rules)
msgs := testTransferTo(t, rules, provider, []string{
"All Mail.mbox/mbox:1",
"All Mail.mbox/mbox:2",
})
got := map[string][]string{}
for _, msg := range msgs {
got[msg.ID] = msg.targetNames()
}
r.Equal(t, map[string][]string{
"All Mail.mbox/mbox:1": {"Archive", "Foo"}, // Bar is not in rules.
"All Mail.mbox/mbox:2": {"Archive", "Foo"},
}, got)
}
func TestMBOXProviderTransferFrom(t *testing.T) {
dir, err := ioutil.TempDir("", "mbox")
r.NoError(t, err)
defer os.RemoveAll(dir) //nolint[errcheck]
provider := newTestMBOXProvider(dir)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupMBOXRules(rules)
testTransferFrom(t, rules, provider, []Message{
{ID: "Foo.mbox:1", Body: getTestMsgBody("msg"), Targets: []Mailbox{{Name: "Foo"}}},
})
checkMBOXFileStructure(t, dir, []string{
"Foo.mbox",
})
}
func TestMBOXProviderTransferFromTo(t *testing.T) {
dir, err := ioutil.TempDir("", "mbox")
r.NoError(t, err)
defer os.RemoveAll(dir) //nolint[errcheck]
source := newTestMBOXProvider("")
target := newTestMBOXProvider(dir)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupMBOXRules(rules)
testTransferFromTo(t, rules, source, target, 5*time.Second)
checkMBOXFileStructure(t, dir, []string{
"Archive.mbox",
"Foo.mbox",
"Inbox.mbox",
})
}
func TestMBOXProviderGetMessageRules(t *testing.T) {
provider := newTestMBOXProvider("")
body := []byte(`Subject: Test
X-Gmail-Labels: foo,bar
`)
rules := transferRules{
rules: map[string]*Rule{
"1": {Active: true, SourceMailbox: Mailbox{Name: "folder"}},
"2": {Active: false, SourceMailbox: Mailbox{Name: "foo"}},
"3": {Active: true, SourceMailbox: Mailbox{Name: "bar"}},
"4": {Active: false, SourceMailbox: Mailbox{Name: "baz"}},
"5": {Active: true, SourceMailbox: Mailbox{Name: "other"}},
},
}
gotRules := provider.getMessageRules(rules, "folder", "id", body)
r.Equal(t, 2, len(gotRules))
r.Equal(t, "folder", gotRules[0].SourceMailbox.Name)
r.Equal(t, "bar", gotRules[1].SourceMailbox.Name)
}
func TestMBOXProviderGetMessageTargetsReturnsOnlyOneFolder(t *testing.T) {
provider := newTestMBOXProvider("")
folderA := Mailbox{Name: "Folder A", IsExclusive: true}
folderB := Mailbox{Name: "Folder B", IsExclusive: true}
labelA := Mailbox{Name: "Label A", IsExclusive: false}
labelB := Mailbox{Name: "Label B", IsExclusive: false}
labelC := Mailbox{Name: "Label C", IsExclusive: false}
rule1 := &Rule{TargetMailboxes: []Mailbox{folderA, labelA, labelB}}
rule2 := &Rule{TargetMailboxes: []Mailbox{folderB, labelC}}
rule3 := &Rule{TargetMailboxes: []Mailbox{folderB}}
tests := []struct {
rules []*Rule
wantMailboxes []Mailbox
}{
{[]*Rule{}, []Mailbox{}},
{[]*Rule{rule1}, []Mailbox{folderA, labelA, labelB}},
{[]*Rule{rule1, rule2}, []Mailbox{folderA, labelA, labelB, labelC}},
{[]*Rule{rule1, rule3}, []Mailbox{folderA, labelA, labelB}},
{[]*Rule{rule3, rule1}, []Mailbox{folderB, labelA, labelB}},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.rules), func(t *testing.T) {
mailboxes := provider.getMessageTargets(tc.rules, "", []byte(""))
r.Equal(t, tc.wantMailboxes, mailboxes)
})
}
}
func setupMBOXRules(rules transferRules) {
_ = rules.setRule(Mailbox{Name: "All Mail"}, []Mailbox{{Name: "Archive"}}, 0, 0)
_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
}
func checkMBOXFileStructure(t *testing.T, root string, expectedFiles []string) {
files, err := getAllPathsWithSuffix(root, ".mbox")
r.NoError(t, err)
r.Equal(t, expectedFiles, files)
}

169
internal/transfer/provider_pmapi.go
View File

@ -1,169 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"context"
"sort"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/pkg/errors"
)
const (
fetchWorkers = 20 // In how many workers to fetch message (group list on IMAP).
attachWorkers = 5 // In how many workers to fetch attachments (for one message).
buildWorkers = 20 // In how many workers to build messages.
)
// PMAPIProvider implements import and export to/from ProtonMail server.
type PMAPIProvider struct {
client pmapi.Client
userID string
addressID string
keyRing *crypto.KeyRing
builder *message.Builder
nextImportRequests map[string]*pmapi.ImportMsgReq // Key is msg transfer ID.
nextImportRequestsSize int
timeIt *timeIt
connection bool
}
// NewPMAPIProvider returns new PMAPIProvider.
func NewPMAPIProvider(client pmapi.Client, userID, addressID string) (*PMAPIProvider, error) {
provider := &PMAPIProvider{
client: client,
userID: userID,
addressID: addressID,
builder: message.NewBuilder(fetchWorkers, attachWorkers, buildWorkers),
nextImportRequests: map[string]*pmapi.ImportMsgReq{},
nextImportRequestsSize: 0,
timeIt: newTimeIt("pmapi"),
}
if addressID != "" {
keyRing, err := client.KeyRingForAddressID(addressID)
if err != nil {
return nil, errors.Wrap(err, "failed to get key ring")
}
provider.keyRing = keyRing
}
return provider, nil
}
// ID returns identifier of current setup of PMAPI provider.
// Identification is unique per user.
func (p *PMAPIProvider) ID() string {
return p.userID
}
// Mailboxes returns all available labels in ProtonMail account.
func (p *PMAPIProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
labels, err := p.client.ListLabels(context.Background())
if err != nil {
return nil, err
}
sortedLabels := byFoldersLabels(labels)
sort.Sort(sortedLabels)
emptyLabelsMap := map[string]bool{}
if !includeEmpty {
messagesCounts, err := p.client.CountMessages(context.Background(), p.addressID)
if err != nil {
return nil, err
}
for _, messagesCount := range messagesCounts {
if messagesCount.Total == 0 {
emptyLabelsMap[messagesCount.LabelID] = true
}
}
}
mailboxes := []Mailbox{}
for _, mailbox := range getSystemMailboxes(includeAllMail) {
if !includeEmpty && emptyLabelsMap[mailbox.ID] {
continue
}
mailboxes = append(mailboxes, mailbox)
}
for _, label := range sortedLabels {
if !includeEmpty && emptyLabelsMap[label.ID] {
continue
}
mailboxes = append(mailboxes, Mailbox{
ID: label.ID,
Name: label.Name,
Color: label.Color,
IsExclusive: bool(label.Exclusive),
})
}
return mailboxes, nil
}
func getSystemMailboxes(includeAllMail bool) []Mailbox {
mailboxes := []Mailbox{
{ID: pmapi.InboxLabel, Name: "Inbox", IsExclusive: true},
{ID: pmapi.DraftLabel, Name: "Drafts", IsExclusive: true},
{ID: pmapi.SentLabel, Name: "Sent", IsExclusive: true},
{ID: pmapi.StarredLabel, Name: "Starred", IsExclusive: true},
{ID: pmapi.ArchiveLabel, Name: "Archive", IsExclusive: true},
{ID: pmapi.SpamLabel, Name: "Spam", IsExclusive: true},
{ID: pmapi.TrashLabel, Name: "Trash", IsExclusive: true},
}
if includeAllMail {
mailboxes = append(mailboxes, Mailbox{
ID: pmapi.AllMailLabel,
Name: "All Mail",
IsExclusive: true,
})
}
return mailboxes
}
type byFoldersLabels []*pmapi.Label
func (l byFoldersLabels) Len() int {
return len(l)
}
func (l byFoldersLabels) Swap(i, j int) {
l[i], l[j] = l[j], l[i]
}
// Less sorts first folders, then labels, by user order.
func (l byFoldersLabels) Less(i, j int) bool {
if l[i].Exclusive && !l[j].Exclusive {
return true
}
if !l[i].Exclusive && l[j].Exclusive {
return false
}
return l[i].Order < l[j].Order
}

179
internal/transfer/provider_pmapi_source.go
View File

@ -1,179 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"context"
"errors"
"fmt"
"sync"
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/sirupsen/logrus"
)
const pmapiListPageSize = 150
// TransferTo exports messages based on rules to channel.
func (p *PMAPIProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
log.Info("Started transfer from PMAPI to channel")
defer log.Info("Finished transfer from PMAPI to channel")
p.timeIt.clear()
defer p.timeIt.logResults()
// TransferTo cannot end sooner than loadCounts goroutine because
// loadCounts writes to channel in progress which would be closed.
// That can happen for really small accounts.
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
p.loadCounts(rules, progress)
}()
for rule := range rules.iterateActiveRules() {
p.transferTo(rule, progress, ch, rules.skipEncryptedMessages)
}
wg.Wait()
}
func (p *PMAPIProvider) loadCounts(rules transferRules, progress *Progress) {
for rule := range rules.iterateActiveRules() {
if progress.shouldStop() {
break
}
rule := rule
progress.callWrap(func() error {
_, total, err := p.listMessages(&pmapi.MessagesFilter{
AddressID: p.addressID,
LabelID: rule.SourceMailbox.ID,
Begin: rule.FromTime,
End: rule.ToTime,
Limit: 0,
})
if err != nil {
log.WithError(err).Warning("Problem to load counts")
return err
}
progress.updateCount(rule.SourceMailbox.Name, uint(total))
return nil
})
}
progress.countsFinal()
}
func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, skipEncryptedMessages bool) {
page := 0
for {
if progress.shouldStop() {
break
}
isLastPage := true
progress.callWrap(func() error {
// Would be better to filter by Begin and BeginID to be sure
// in case user deletes messages during the process, no message
// is skipped (paging is off then), but API does not support
// filtering by both mentioned fields at the same time.
desc := false
pmapiMessages, total, err := p.listMessages(&pmapi.MessagesFilter{
AddressID: p.addressID,
LabelID: rule.SourceMailbox.ID,
Begin: rule.FromTime,
End: rule.ToTime,
PageSize: pmapiListPageSize,
Page: page,
Sort: "ID",
Desc: &desc,
})
if err != nil {
return err
}
log.WithFields(logrus.Fields{
"label": rule.SourceMailbox.ID,
"page": page,
"total": total,
"count": len(pmapiMessages),
}).Debug("Listing messages")
isLastPage = len(pmapiMessages) < pmapiListPageSize
for _, pmapiMessage := range pmapiMessages {
if progress.shouldStop() {
break
}
msgID := fmt.Sprintf("%s_%s", rule.SourceMailbox.ID, pmapiMessage.ID)
progress.addMessage(msgID, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames())
msg, err := p.exportMessage(rule, progress, pmapiMessage.ID, msgID, skipEncryptedMessages)
progress.messageExported(msgID, msg.Body, err)
if err == nil {
ch <- msg
}
}
page++
return nil
})
if isLastPage {
break
}
}
}
func (p *PMAPIProvider) exportMessage(rule *Rule, progress *Progress, pmapiMsgID, msgID string, skipEncryptedMessages bool) (Message, error) {
var msg *pmapi.Message
progress.callWrap(func() error {
var err error
msg, err = p.getMessage(pmapiMsgID)
return err
})
p.timeIt.start("build", msgID)
defer p.timeIt.stop("build", msgID)
body, err := p.builder.NewJobWithOptions(
context.Background(),
p.client,
msg.ID,
message.JobOptions{IgnoreDecryptionErrors: !skipEncryptedMessages},
).GetResult()
if err != nil {
if errors.Is(err, message.ErrDecryptionFailed) && skipEncryptedMessages {
err = errors.New("skipping encrypted message")
}
return Message{Body: []byte(msg.Body)}, err
}
return Message{
ID: msgID,
Unread: bool(msg.Unread),
Body: body,
Sources: []Mailbox{rule.SourceMailbox},
Targets: rule.TargetMailboxes,
}, nil
}

339
internal/transfer/provider_pmapi_target.go
View File

@ -1,339 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"net/mail"
"sync"
pkgMsg "github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/pkg/errors"
)
const (
pmapiImportBatchMaxItems = 10
pmapiImportBatchMaxSize = 25 * 1000 * 1000 // 25 MB
pmapiImportWorkers = 4 // To keep memory under 1 GB.
)
// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
func (p *PMAPIProvider) DefaultMailboxes(_ Mailbox) []Mailbox {
return []Mailbox{{
ID: pmapi.ArchiveLabel,
Name: "Archive",
IsExclusive: true,
}}
}
// CreateMailbox creates label in ProtonMail account.
func (p *PMAPIProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) {
if mailbox.ID != "" {
return Mailbox{}, errors.New("mailbox is already created")
}
label, err := p.client.CreateLabel(context.Background(), &pmapi.Label{
Name: mailbox.Name,
Color: mailbox.Color,
Exclusive: pmapi.Boolean(mailbox.IsExclusive),
Type: pmapi.LabelTypeMailbox,
})
if err != nil {
return Mailbox{}, errors.Wrap(err, fmt.Sprintf("failed to create mailbox %s", mailbox.Name))
}
mailbox.ID = label.ID
return mailbox, nil
}
// TransferFrom imports messages from channel.
func (p *PMAPIProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) {
log.Info("Started transfer from channel to PMAPI")
defer log.Info("Finished transfer from channel to PMAPI")
p.timeIt.clear()
defer p.timeIt.logResults()
// Cache has to be cleared before each transfer to not contain
// old stuff from previous cancelled run.
p.nextImportRequests = map[string]*pmapi.ImportMsgReq{}
p.nextImportRequestsSize = 0
preparedImportRequestsCh := make(chan map[string]*pmapi.ImportMsgReq)
wg := p.startImportWorkers(progress, preparedImportRequestsCh)
for msg := range ch {
if progress.shouldStop() {
break
}
if p.isMessageDraft(msg) {
p.transferDraft(rules, progress, msg)
} else {
p.transferMessage(rules, progress, msg, preparedImportRequestsCh)
}
}
if len(p.nextImportRequests) > 0 {
preparedImportRequestsCh <- p.nextImportRequests
}
close(preparedImportRequestsCh)
wg.Wait()
}
func (p *PMAPIProvider) isMessageDraft(msg Message) bool {
for _, target := range msg.Targets {
if target.ID == pmapi.DraftLabel {
return true
}
}
return false
}
func (p *PMAPIProvider) transferDraft(rules transferRules, progress *Progress, msg Message) {
importedID, err := p.importDraft(msg, rules.globalMailbox)
progress.messageImported(msg.ID, importedID, err)
}
func (p *PMAPIProvider) importDraft(msg Message, globalMailbox *Mailbox) (string, error) { //nolint[funlen]
message, attachmentReaders, err := p.parseMessage(msg)
if err != nil {
return "", errors.Wrap(err, "failed to parse message")
}
if message.Sender == nil {
mainAddress := p.client.Addresses().Main()
message.Sender = &mail.Address{
Name: mainAddress.DisplayName,
Address: mainAddress.Email,
}
}
// Trying to encrypt an encrypted draft will return an error;
// users are forbidden to import messages encrypted with foreign keys to drafts.
if message.IsEncrypted() {
return "", errors.New("refusing to import draft encrypted by foreign key")
}
p.timeIt.start("encrypt", msg.ID)
err = message.Encrypt(p.keyRing, nil)
p.timeIt.stop("encrypt", msg.ID)
if err != nil {
return "", errors.Wrap(err, "failed to encrypt draft")
}
if globalMailbox != nil {
message.LabelIDs = append(message.LabelIDs, globalMailbox.ID)
}
attachments := message.Attachments
message.Attachments = nil
draft, err := p.createDraft(msg.ID, message, "", pmapi.DraftActionReply)
if err != nil {
return "", errors.Wrap(err, "failed to create draft")
}
for idx, attachment := range attachments {
attachment.MessageID = draft.ID
attachmentBody, _ := ioutil.ReadAll(attachmentReaders[idx])
r := bytes.NewReader(attachmentBody)
sigReader, err := attachment.DetachedSign(p.keyRing, r)
if err != nil {
return "", errors.Wrap(err, "failed to sign attachment")
}
p.timeIt.start("encrypt", msg.ID)
r = bytes.NewReader(attachmentBody)
encReader, err := attachment.Encrypt(p.keyRing, r)
p.timeIt.stop("encrypt", msg.ID)
if err != nil {
return "", errors.Wrap(err, "failed to encrypt attachment")
}
_, err = p.createAttachment(msg.ID, attachment, encReader, sigReader)
if err != nil {
return "", errors.Wrap(err, "failed to create attachment")
}
}
return draft.ID, nil
}
func (p *PMAPIProvider) transferMessage(rules transferRules, progress *Progress, msg Message, preparedImportRequestsCh chan map[string]*pmapi.ImportMsgReq) {
importMsgReq, err := p.generateImportMsgReq(rules, progress, msg)
if err != nil {
progress.messageImported(msg.ID, "", err)
return
}
if importMsgReq == nil || progress.shouldStop() {
return
}
importMsgReqSize := len(importMsgReq.Message)
if p.nextImportRequestsSize+importMsgReqSize > pmapiImportBatchMaxSize || len(p.nextImportRequests) == pmapiImportBatchMaxItems {
preparedImportRequestsCh <- p.nextImportRequests
p.nextImportRequests = map[string]*pmapi.ImportMsgReq{}
p.nextImportRequestsSize = 0
}
p.nextImportRequests[msg.ID] = importMsgReq
p.nextImportRequestsSize += importMsgReqSize
}
func (p *PMAPIProvider) generateImportMsgReq(rules transferRules, progress *Progress, msg Message) (*pmapi.ImportMsgReq, error) {
message, attachmentReaders, err := p.parseMessage(msg)
if err != nil {
return nil, errors.Wrap(err, "failed to parse message")
}
var body []byte
if message.IsEncrypted() {
if rules.skipEncryptedMessages {
progress.messageSkipped(msg.ID)
return nil, nil
}
body = msg.Body
} else {
p.timeIt.start("encrypt", msg.ID)
body, err = p.encryptMessage(message, attachmentReaders)
p.timeIt.stop("encrypt", msg.ID)
if err != nil {
return nil, errors.Wrap(err, "failed to encrypt message")
}
}
labelIDs := []string{}
for _, target := range msg.Targets {
// Frontend should not set All Mail to Rules, but to be sure...
if target.ID != pmapi.AllMailLabel {
labelIDs = append(labelIDs, target.ID)
}
}
if rules.globalMailbox != nil {
labelIDs = append(labelIDs, rules.globalMailbox.ID)
}
return &pmapi.ImportMsgReq{
Metadata: &pmapi.ImportMetadata{
AddressID: p.addressID,
Unread: pmapi.Boolean(msg.Unread),
Time: message.Time,
Flags: computeMessageFlags(message.Header),
LabelIDs: labelIDs,
},
Message: body,
}, nil
}
func (p *PMAPIProvider) parseMessage(msg Message) (m *pmapi.Message, r []io.Reader, err error) {
p.timeIt.start("parse", msg.ID)
defer p.timeIt.stop("parse", msg.ID)
message, _, _, attachmentReaders, err := pkgMsg.Parse(bytes.NewBuffer(msg.Body))
return message, attachmentReaders, err
}
func (p *PMAPIProvider) encryptMessage(msg *pmapi.Message, attachmentReaders []io.Reader) ([]byte, error) {
if msg.MIMEType == pmapi.ContentTypeMultipartEncrypted {
return []byte(msg.Body), nil
}
return pkgMsg.BuildEncrypted(msg, attachmentReaders, p.keyRing)
}
func computeMessageFlags(header mail.Header) (flag int64) {
if header.Get("received") == "" {
return pmapi.FlagSent
}
return pmapi.FlagReceived
}
func (p *PMAPIProvider) startImportWorkers(progress *Progress, preparedImportRequestsCh chan map[string]*pmapi.ImportMsgReq) *sync.WaitGroup {
var wg sync.WaitGroup
wg.Add(pmapiImportWorkers)
for i := 0; i < pmapiImportWorkers; i++ {
go func() {
for importRequests := range preparedImportRequestsCh {
p.importMessages(progress, importRequests)
}
wg.Done()
}()
}
return &wg
}
func (p *PMAPIProvider) importMessages(progress *Progress, importRequests map[string]*pmapi.ImportMsgReq) {
if progress.shouldStop() {
return
}
importMsgIDs := []string{}
importMsgRequests := pmapi.ImportMsgReqs{}
for msgID, req := range importRequests {
importMsgIDs = append(importMsgIDs, msgID)
importMsgRequests = append(importMsgRequests, req)
}
log.WithField("msgIDs", importMsgIDs).Trace("Importing messages")
results, err := p.importRequest(importMsgIDs[0], importMsgRequests)
// In case the whole request failed, try to import every message one by one.
if err != nil || len(results) == 0 {
log.WithError(err).Warning("Importing messages failed, trying one by one")
for msgID, req := range importRequests {
importedID, err := p.importMessage(msgID, progress, req)
progress.messageImported(msgID, importedID, err)
}
return
}
// In case request passed but some messages failed, try to import the failed ones alone.
for index, result := range results {
msgID := importMsgIDs[index]
if result.Error != nil {
log.WithError(result.Error).WithField("msg", msgID).Warning("Importing message failed, trying alone")
req := importMsgRequests[index]
importedID, err := p.importMessage(msgID, progress, req)
progress.messageImported(msgID, importedID, err)
} else {
progress.messageImported(msgID, result.MessageID, nil)
}
}
}
func (p *PMAPIProvider) importMessage(msgSourceID string, progress *Progress, req *pmapi.ImportMsgReq) (importedID string, importedErr error) {
progress.callWrap(func() error {
results, err := p.importRequest(msgSourceID, pmapi.ImportMsgReqs{req})
if err != nil {
return errors.Wrap(err, "failed to import messages")
}
if len(results) == 0 {
importedErr = errors.New("import ended with no result")
return nil // This should not happen, only when there is bug which means we should skip this one.
}
if results[0].Error != nil {
importedErr = errors.Wrap(results[0].Error, "failed to import message")
return nil //nolint[nilerr] Call passed but API refused this message, skip this one.
}
importedID = results[0].MessageID
return nil
})
return importedID, importedErr
}

201
internal/transfer/provider_pmapi_test.go
View File

@ -1,201 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"bytes"
"context"
"fmt"
"testing"
"time"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
gomock "github.com/golang/mock/gomock"
r "github.com/stretchr/testify/require"
)
func TestPMAPIProviderMailboxes(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
setupPMAPIClientExpectationForExport(&m)
provider, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID")
r.NoError(t, err)
tests := []struct {
includeEmpty bool
includeAllMail bool
wantMailboxes []Mailbox
}{
{true, false, []Mailbox{
{ID: "folder1", Name: "One", Color: "red", IsExclusive: true},
{ID: "folder2", Name: "Two", Color: "orange", IsExclusive: true},
{ID: "label2", Name: "Bar", Color: "green", IsExclusive: false},
{ID: "label1", Name: "Foo", Color: "blue", IsExclusive: false},
}},
{false, true, []Mailbox{
{ID: pmapi.AllMailLabel, Name: "All Mail", IsExclusive: true},
{ID: "folder1", Name: "One", Color: "red", IsExclusive: true},
{ID: "folder2", Name: "Two", Color: "orange", IsExclusive: true},
{ID: "label1", Name: "Foo", Color: "blue", IsExclusive: false},
}},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v-%v", tc.includeEmpty, tc.includeAllMail), func(t *testing.T) {
mailboxes, err := provider.Mailboxes(tc.includeEmpty, tc.includeAllMail)
r.NoError(t, err)
r.Equal(t, []Mailbox{
{ID: pmapi.InboxLabel, Name: "Inbox", IsExclusive: true},
{ID: pmapi.DraftLabel, Name: "Drafts", IsExclusive: true},
{ID: pmapi.SentLabel, Name: "Sent", IsExclusive: true},
{ID: pmapi.StarredLabel, Name: "Starred", IsExclusive: true},
{ID: pmapi.ArchiveLabel, Name: "Archive", IsExclusive: true},
{ID: pmapi.SpamLabel, Name: "Spam", IsExclusive: true},
{ID: pmapi.TrashLabel, Name: "Trash", IsExclusive: true},
}, mailboxes[:7])
r.Equal(t, tc.wantMailboxes, mailboxes[7:])
})
}
}
func TestPMAPIProviderTransferTo(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
setupPMAPIClientExpectationForExport(&m)
provider, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID")
r.NoError(t, err)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupPMAPIRules(rules)
testTransferTo(t, rules, provider, []string{
"0_msg1",
"0_msg2",
})
}
func TestPMAPIProviderTransferFrom(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
setupPMAPIClientExpectationForImport(&m)
provider, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID")
r.NoError(t, err)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupPMAPIRules(rules)
testTransferFrom(t, rules, provider, []Message{
{ID: "msg1", Body: getTestMsgBody("msg1"), Targets: []Mailbox{{ID: pmapi.InboxLabel}}},
{ID: "msg2", Body: getTestMsgBody("msg2"), Targets: []Mailbox{{ID: pmapi.InboxLabel}}},
})
}
func TestPMAPIProviderTransferFromDraft(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
setupPMAPIClientExpectationForImportDraft(&m)
provider, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID")
r.NoError(t, err)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupPMAPIRules(rules)
testTransferFrom(t, rules, provider, []Message{
{ID: "draft1", Body: getTestMsgBody("draft1"), Targets: []Mailbox{{ID: pmapi.DraftLabel}}},
})
}
func TestPMAPIProviderTransferFromTo(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
setupPMAPIClientExpectationForExport(&m)
setupPMAPIClientExpectationForImport(&m)
source, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID")
r.NoError(t, err)
target, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID")
r.NoError(t, err)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupPMAPIRules(rules)
testTransferFromTo(t, rules, source, target, 5*time.Second)
}
func setupPMAPIRules(rules transferRules) {
_ = rules.setRule(Mailbox{ID: pmapi.InboxLabel}, []Mailbox{{ID: pmapi.InboxLabel}}, 0, 0)
}
func setupPMAPIClientExpectationForExport(m *mocks) {
m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes()
m.pmapiClient.EXPECT().ListLabels(gomock.Any()).Return([]*pmapi.Label{
{ID: "label1", Name: "Foo", Color: "blue", Exclusive: false, Order: 2},
{ID: "label2", Name: "Bar", Color: "green", Exclusive: false, Order: 1},
{ID: "folder1", Name: "One", Color: "red", Exclusive: true, Order: 1},
{ID: "folder2", Name: "Two", Color: "orange", Exclusive: true, Order: 2},
}, nil).AnyTimes()
m.pmapiClient.EXPECT().CountMessages(gomock.Any(), gomock.Any()).Return([]*pmapi.MessagesCount{
{LabelID: "label1", Total: 10},
{LabelID: "label2", Total: 0},
{LabelID: "folder1", Total: 20},
}, nil).AnyTimes()
m.pmapiClient.EXPECT().ListMessages(gomock.Any(), gomock.Any()).Return([]*pmapi.Message{
{ID: "msg1"},
{ID: "msg2"},
}, 2, nil).AnyTimes()
m.pmapiClient.EXPECT().GetMessage(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, msgID string) (*pmapi.Message, error) {
return &pmapi.Message{
ID: msgID,
Body: string(getTestMsgBody(msgID)),
MIMEType: pmapi.ContentTypeMultipartMixed,
}, nil
}).AnyTimes()
}
func setupPMAPIClientExpectationForImport(m *mocks) {
m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes()
m.pmapiClient.EXPECT().Import(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, requests pmapi.ImportMsgReqs) ([]*pmapi.ImportMsgRes, error) {
results := []*pmapi.ImportMsgRes{}
for _, request := range requests {
for _, msgID := range []string{"msg1", "msg2"} {
if bytes.Contains(request.Message, []byte(msgID)) {
results = append(results, &pmapi.ImportMsgRes{MessageID: msgID, Error: nil})
}
}
}
return results, nil
}).AnyTimes()
}
func setupPMAPIClientExpectationForImportDraft(m *mocks) {
m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes()
m.pmapiClient.EXPECT().CreateDraft(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, msg *pmapi.Message, parentID string, action int) (*pmapi.Message, error) {
r.Equal(m.t, msg.Subject, "draft1")
msg.ID = "draft1"
return msg, nil
})
}

136
internal/transfer/provider_pmapi_utils.go
View File

@ -1,136 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"context"
"fmt"
"io"
"time"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/pkg/errors"
)
const (
pmapiRetries = 10
pmapiReconnectTimeout = 30 * time.Minute
pmapiReconnectSleep = 10 * time.Second
)
func (p *PMAPIProvider) SetConnectionUp() {
p.connection = true
}
func (p *PMAPIProvider) SetConnectionDown() {
p.connection = false
}
func (p *PMAPIProvider) ensureConnection(callback func() error) error {
var callErr error
for i := 1; i <= pmapiRetries; i++ {
callErr = callback()
if callErr == nil {
return nil
}
log.WithField("attempt", i).WithError(callErr).Warning("API call failed, trying reconnect")
err := p.tryReconnect()
if err != nil {
return err
}
}
return errors.Wrap(callErr, "too many retries")
}
func (p *PMAPIProvider) tryReconnect() error {
start := time.Now()
var previousErr error
for {
if time.Since(start) > pmapiReconnectTimeout {
return previousErr
}
if !p.connection {
time.Sleep(pmapiReconnectSleep)
continue
}
break
}
return nil
}
func (p *PMAPIProvider) listMessages(filter *pmapi.MessagesFilter) (messages []*pmapi.Message, count int, err error) {
err = p.ensureConnection(func() error {
// Sort is used in the key so the filter is different for estimating and real fetching.
key := fmt.Sprintf("%s_%s_%d", filter.LabelID, filter.Sort, filter.Page)
p.timeIt.start("listing", key)
defer p.timeIt.stop("listing", key)
messages, count, err = p.client.ListMessages(context.Background(), filter)
return err
})
return
}
func (p *PMAPIProvider) getMessage(msgID string) (message *pmapi.Message, err error) {
err = p.ensureConnection(func() error {
p.timeIt.start("download", msgID)
defer p.timeIt.stop("download", msgID)
message, err = p.client.GetMessage(context.Background(), msgID)
return err
})
return
}
func (p *PMAPIProvider) importRequest(msgSourceID string, req pmapi.ImportMsgReqs) (res []*pmapi.ImportMsgRes, err error) {
err = p.ensureConnection(func() error {
p.timeIt.start("upload", msgSourceID)
defer p.timeIt.stop("upload", msgSourceID)
res, err = p.client.Import(context.Background(), req)
return err
})
return
}
func (p *PMAPIProvider) createDraft(msgSourceID string, message *pmapi.Message, parent string, action int) (draft *pmapi.Message, err error) {
err = p.ensureConnection(func() error {
p.timeIt.start("upload", msgSourceID)
defer p.timeIt.stop("upload", msgSourceID)
draft, err = p.client.CreateDraft(context.Background(), message, parent, action)
return err
})
return
}
func (p *PMAPIProvider) createAttachment(msgSourceID string, att *pmapi.Attachment, r io.Reader, sig io.Reader) (created *pmapi.Attachment, err error) {
err = p.ensureConnection(func() error {
// Use some attributes from attachment to have unique key for each call.
key := fmt.Sprintf("%s_%s_%d", msgSourceID, att.Name, att.Size)
p.timeIt.start("upload", key)
defer p.timeIt.stop("upload", key)
created, err = p.client.CreateAttachment(context.Background(), att, r, sig)
return err
})
return
}

115
internal/transfer/provider_test.go
View File

@ -1,115 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"testing"
"time"
a "github.com/stretchr/testify/assert"
r "github.com/stretchr/testify/require"
)
func getTestMsgBody(subject string) []byte {
return []byte(fmt.Sprintf(`Subject: %s
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
Content-Type: multipart/mixed; boundary=c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a
--c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a
Content-Disposition: inline
Content-Transfer-Encoding: 7bit
Content-Type: text/plain; charset=utf-8
hello
--c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a--
`, subject))
}
func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, expectedMessageIDs []string) []Message {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
ch := make(chan Message)
go func() {
provider.TransferTo(rules, &progress, ch)
close(ch)
}()
msgs := []Message{}
gotMessageIDs := []string{}
for msg := range ch {
msgs = append(msgs, msg)
gotMessageIDs = append(gotMessageIDs, msg.ID)
}
r.ElementsMatch(t, expectedMessageIDs, gotMessageIDs)
r.Empty(t, progress.GetFailedMessages())
return msgs
}
func testTransferFrom(t *testing.T, rules transferRules, provider TargetProvider, messages []Message) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
ch := make(chan Message)
go func() {
for _, message := range messages {
progress.addMessage(message.ID, []string{}, []string{})
progress.messageExported(message.ID, []byte(""), nil)
ch <- message
}
close(ch)
}()
go func() {
provider.TransferFrom(rules, &progress, ch)
progress.finish()
}()
maxWait := time.Duration(len(messages)*2) * time.Second
a.Eventually(t, func() bool {
return progress.updateCh == nil
}, maxWait, 10*time.Millisecond, "Waiting for imported messages timed out")
r.Empty(t, progress.GetFailedMessages())
}
func testTransferFromTo(t *testing.T, rules transferRules, source SourceProvider, target TargetProvider, maxWait time.Duration) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
ch := make(chan Message)
go func() {
source.TransferTo(rules, &progress, ch)
close(ch)
}()
go func() {
target.TransferFrom(rules, &progress, ch)
progress.finish()
}()
a.Eventually(t, func() bool {
return progress.updateCh == nil
}, maxWait, 10*time.Millisecond, "Waiting for export and import timed out")
r.Empty(t, progress.GetFailedMessages())
}

145
internal/transfer/report.go
View File

@ -1,145 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"bytes"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/pkg/errors"
)
// fileReport is struct which can write and read message details.
// File report includes private information.
type fileReport struct {
path string
}
func openLastFileReport(reportsPath, importID string) (*fileReport, error) { //nolint[deadcode]
allLogFileNames, err := getFilePathsWithSuffix(reportsPath, ".log")
if err != nil {
return nil, err
}
reportFileNames := []string{}
for _, fileName := range allLogFileNames {
if strings.HasPrefix(fileName, fmt.Sprintf("import_%s_", importID)) {
reportFileNames = append(reportFileNames, fileName)
}
}
if len(reportFileNames) == 0 {
return nil, errors.New("no report found")
}
sort.Strings(reportFileNames)
reportFileName := reportFileNames[len(reportFileNames)-1]
path := filepath.Join(reportsPath, reportFileName)
return &fileReport{
path: path,
}, nil
}
func newFileReport(reportsPath, importID string) *fileReport {
fileName := fmt.Sprintf("import_%s_%d.log", importID, time.Now().Unix())
path := filepath.Join(reportsPath, fileName)
return &fileReport{
path: path,
}
}
func (r *fileReport) writeMessageStatus(messageStatus *MessageStatus) {
f, err := os.OpenFile(r.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
log.WithError(err).Error("Failed to open report file")
}
defer f.Close() //nolint[errcheck]
messageReport := newMessageReportFromMessageStatus(messageStatus, true)
data, err := json.Marshal(messageReport)
if err != nil {
log.WithError(err).Error("Failed to marshall message details")
}
data = append(data, '\n')
if _, err = f.Write(data); err != nil {
log.WithError(err).Error("Failed to write to report file")
}
}
// bugReport is struct which can create report for bug reporting.
// Bug report does NOT include private information.
type bugReport struct {
data bytes.Buffer
}
func (r *bugReport) writeMessageStatus(messageStatus *MessageStatus) {
messageReport := newMessageReportFromMessageStatus(messageStatus, false)
data, err := json.Marshal(messageReport)
if err != nil {
log.WithError(err).Error("Failed to marshall message details")
}
_, _ = r.data.Write(data)
_, _ = r.data.Write([]byte("\n"))
}
func (r *bugReport) getData() []byte {
return r.data.Bytes()
}
// messageReport is struct which holds data used by `fileReport` and `bugReport`.
type messageReport struct {
EventTime int64
SourceID string
TargetID string
BodyHash string
SourceMailboxes []string
TargetMailboxes []string
Error string
// Private information for user.
Subject string
From string
Time string
}
func newMessageReportFromMessageStatus(messageStatus *MessageStatus, includePrivateInfo bool) messageReport {
md := messageReport{
EventTime: messageStatus.eventTime.Unix(),
SourceID: messageStatus.SourceID,
TargetID: messageStatus.targetID,
BodyHash: messageStatus.bodyHash,
SourceMailboxes: messageStatus.sourceNames,
TargetMailboxes: messageStatus.targetNames,
Error: messageStatus.GetErrorMessage(),
}
if includePrivateInfo {
md.Subject = messageStatus.Subject
md.From = messageStatus.From
md.Time = messageStatus.Time.Format(time.RFC1123Z)
}
return md
}

366
internal/transfer/rules.go
View File

@ -1,366 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/pkg/errors"
)
// transferRules maintains import rules, e.g. to which target mailbox should be
// source mailbox imported or what time spans.
type transferRules struct {
filePath string
// rules is map with key as hash of source mailbox to its rule.
// Every source mailbox should have rule, at least disabled one.
rules map[string]*Rule
// globalMailbox is applied to every message in the import phase.
// E.g., every message will be imported into this mailbox.
globalMailbox *Mailbox
// globalFromTime and globalToTime is applied to every rule right
// before the transfer (propagateGlobalTime has to be called).
globalFromTime int64
globalToTime int64
// skipEncryptedMessages determines whether message which cannot
// be decrypted should be imported/exported or skipped.
skipEncryptedMessages bool
}
// loadRules loads rules from `rulesPath` based on `ruleID`.
func loadRules(rulesPath, ruleID string) transferRules {
fileName := fmt.Sprintf("rules_%s.json", ruleID)
filePath := filepath.Join(rulesPath, fileName)
var rules map[string]*Rule
f, err := os.Open(filePath) //nolint[gosec]
if err != nil {
log.WithError(err).Debug("Problem to read rules")
} else {
defer f.Close() //nolint[errcheck]
if err := json.NewDecoder(f).Decode(&rules); err != nil {
log.WithError(err).Warn("Problem to umarshal rules")
}
}
if rules == nil {
rules = map[string]*Rule{}
}
return transferRules{
filePath: filePath,
rules: rules,
}
}
func (r *transferRules) setSkipEncryptedMessages(skip bool) {
r.skipEncryptedMessages = skip
}
func (r *transferRules) setGlobalMailbox(mailbox *Mailbox) {
r.globalMailbox = mailbox
}
func (r *transferRules) setGlobalTimeLimit(fromTime, toTime int64) {
r.globalFromTime = fromTime
r.globalToTime = toTime
}
func (r *transferRules) propagateGlobalTime() {
if r.globalFromTime == 0 && r.globalToTime == 0 {
return
}
for _, rule := range r.rules {
if !rule.HasTimeLimit() {
rule.FromTime = r.globalFromTime
rule.ToTime = r.globalToTime
}
}
}
func (r *transferRules) getRuleBySourceMailboxName(name string) (*Rule, error) {
for _, rule := range r.rules {
if rule.SourceMailbox.Name == name {
return rule, nil
}
}
return nil, fmt.Errorf("no rule for mailbox %s", name)
}
func (r *transferRules) iterateActiveRules() chan *Rule {
ch := make(chan *Rule)
go func() {
for _, rule := range r.rules {
if rule.Active {
ch <- rule
}
}
close(ch)
}()
return ch
}
// setDefaultRules iterates `sourceMailboxes` and sets missing rules with
// matching mailboxes from `targetMailboxes`. In case no matching mailbox
// is found, `defaultCallback` with a source mailbox as a parameter is used.
func (r *transferRules) setDefaultRules(sourceMailboxes []Mailbox, targetMailboxes []Mailbox, defaultCallback func(Mailbox) []Mailbox) {
for _, sourceMailbox := range sourceMailboxes {
h := sourceMailbox.Hash()
if _, ok := r.rules[h]; ok {
continue
}
targetMailboxes := sourceMailbox.findMatchingMailboxes(targetMailboxes)
if !containsExclusive(targetMailboxes) {
targetMailboxes = append(targetMailboxes, defaultCallback(sourceMailbox)...)
}
active := true
if len(targetMailboxes) == 0 {
active = false
}
// For both import to or export from ProtonMail, spam and draft
// mailboxes are by default deactivated.
for _, mailbox := range append([]Mailbox{sourceMailbox}, targetMailboxes...) {
if mailbox.ID == pmapi.SpamLabel || mailbox.ID == pmapi.DraftLabel || mailbox.ID == pmapi.TrashLabel {
active = false
break
}
}
r.rules[h] = &Rule{
Active: active,
SourceMailbox: sourceMailbox,
TargetMailboxes: targetMailboxes,
}
}
// There is no point showing rule which has no action (i.e., source mailbox
// is not available).
// A good reason to keep all rules and only deactivate them would be for
// multiple imports from different sources with the same or similar enough
// mailbox setup to reuse configuration. That is very minor feature which
// can be implemented in more reasonable way by allowing users to save and
// load configurations.
for key, rule := range r.rules {
found := false
for _, sourceMailbox := range sourceMailboxes {
if sourceMailbox.Name == rule.SourceMailbox.Name {
found = true
}
}
if !found {
delete(r.rules, key)
}
}
r.save()
}
// setRule sets messages from `sourceMailbox` between `fromData` and `toDate`
// (if used) to be imported to all `targetMailboxes`.
func (r *transferRules) setRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error {
numberOfExclusiveMailboxes := 0
for _, mailbox := range targetMailboxes {
if mailbox.IsExclusive {
numberOfExclusiveMailboxes++
}
}
if numberOfExclusiveMailboxes > 1 {
return errors.New("rule can have only one exclusive target mailbox")
}
h := sourceMailbox.Hash()
r.rules[h] = &Rule{
Active: true,
SourceMailbox: sourceMailbox,
TargetMailboxes: targetMailboxes,
FromTime: fromTime,
ToTime: toTime,
}
r.save()
return nil
}
// unsetRule unsets messages from `sourceMailbox` to be exported.
func (r *transferRules) unsetRule(sourceMailbox Mailbox) {
h := sourceMailbox.Hash()
if rule, ok := r.rules[h]; ok {
rule.Active = false
} else {
r.rules[h] = &Rule{
Active: false,
SourceMailbox: sourceMailbox,
}
}
r.save()
}
// getRule returns rule for `sourceMailbox` or nil if it does not exist.
func (r *transferRules) getRule(sourceMailbox Mailbox) *Rule {
h := sourceMailbox.Hash()
return r.rules[h]
}
// getSortedRules returns all set rules in order by `byRuleOrder`.
func (r *transferRules) getSortedRules() []*Rule {
rules := []*Rule{}
for _, rule := range r.rules {
rules = append(rules, rule)
}
sort.Sort(byRuleOrder(rules))
return rules
}
// reset wipes our all rules.
func (r *transferRules) reset() {
r.rules = map[string]*Rule{}
r.save()
}
// save saves rules to file.
func (r *transferRules) save() {
f, err := os.Create(r.filePath)
if err != nil {
log.WithError(err).Warn("Problem to write rules")
return
}
defer f.Close() //nolint[errcheck]
if err := json.NewEncoder(f).Encode(r.rules); err != nil {
log.WithError(err).Warn("Problem to marshal rules")
}
}
// Rule is data holder of rule for one source mailbox used by `transferRules`.
type Rule struct {
Active bool `json:"active"`
SourceMailbox Mailbox `json:"source"`
TargetMailboxes []Mailbox `json:"targets"`
FromTime int64 `json:"from"`
ToTime int64 `json:"to"`
}
// String returns textual representation for log purposes.
func (r *Rule) String() string {
return fmt.Sprintf(
"%s -> %s (%d - %d)",
r.SourceMailbox.Name,
strings.Join(r.TargetMailboxNames(), ", "),
r.FromTime,
r.ToTime,
)
}
func (r *Rule) isTimeInRange(t int64) bool {
if !r.HasTimeLimit() {
return true
}
return r.FromTime <= t && t <= r.ToTime
}
// HasTimeLimit returns whether rule defines time limit.
func (r *Rule) HasTimeLimit() bool {
return r.FromTime != 0 || r.ToTime != 0
}
// FromDate returns time struct based on `FromTime`.
func (r *Rule) FromDate() time.Time {
return time.Unix(r.FromTime, 0)
}
// ToDate returns time struct based on `ToTime`.
func (r *Rule) ToDate() time.Time {
return time.Unix(r.ToTime, 0)
}
// TargetMailboxNames returns array of target mailbox names.
func (r *Rule) TargetMailboxNames() (names []string) {
for _, mailbox := range r.TargetMailboxes {
names = append(names, mailbox.Name)
}
return
}
// byRuleOrder implements sort.Interface. Sort order:
// * System folders first (as defined in getSystemMailboxes).
// * Custom folders by name.
// * Custom labels by name.
type byRuleOrder []*Rule
func (a byRuleOrder) Len() int {
return len(a)
}
func (a byRuleOrder) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}
func (a byRuleOrder) Less(i, j int) bool {
if a[i].SourceMailbox.IsExclusive && !a[j].SourceMailbox.IsExclusive {
return true
}
if !a[i].SourceMailbox.IsExclusive && a[j].SourceMailbox.IsExclusive {
return false
}
iSystemIndex := -1
jSystemIndex := -1
for index, systemFolders := range getSystemMailboxes(true) {
if a[i].SourceMailbox.Name == systemFolders.Name {
iSystemIndex = index
}
if a[j].SourceMailbox.Name == systemFolders.Name {
jSystemIndex = index
}
}
if iSystemIndex != -1 && jSystemIndex == -1 {
return true
}
if iSystemIndex == -1 && jSystemIndex != -1 {
return false
}
if iSystemIndex != -1 && jSystemIndex != -1 {
return iSystemIndex < jSystemIndex
}
return a[i].SourceMailbox.Name < a[j].SourceMailbox.Name
}
// containsExclusive returns true if there is at least one exclusive mailbox.
func containsExclusive(mailboxes []Mailbox) bool {
for _, m := range mailboxes {
if m.IsExclusive {
return true
}
}
return false
}

247
internal/transfer/rules_test.go
View File

@ -1,247 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"io/ioutil"
"os"
"testing"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
r "github.com/stretchr/testify/require"
)
func newTestRules(t *testing.T) (transferRules, func()) {
path, err := ioutil.TempDir("", "rules")
r.NoError(t, err)
ruleID := "rule"
rules := loadRules(path, ruleID)
return rules, func() {
_ = os.RemoveAll(path)
}
}
func TestLoadRules(t *testing.T) {
path, err := ioutil.TempDir("", "rules")
r.NoError(t, err)
defer os.RemoveAll(path) //nolint[errcheck]
ruleID := "rule"
rules := loadRules(path, ruleID)
mailboxA := Mailbox{ID: "1", Name: "One", Color: "orange", IsExclusive: true}
mailboxB := Mailbox{ID: "2", Name: "Two", Color: "", IsExclusive: true}
mailboxC := Mailbox{ID: "3", Name: "Three", Color: "", IsExclusive: false}
r.NoError(t, rules.setRule(mailboxA, []Mailbox{mailboxB, mailboxC}, 0, 0))
r.NoError(t, rules.setRule(mailboxB, []Mailbox{mailboxB}, 10, 20))
r.NoError(t, rules.setRule(mailboxC, []Mailbox{}, 0, 30))
rules2 := loadRules(path, ruleID)
r.Equal(t, map[string]*Rule{
mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB, mailboxC}, FromTime: 0, ToTime: 0},
mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 10, ToTime: 20},
mailboxC.Hash(): {Active: true, SourceMailbox: mailboxC, TargetMailboxes: []Mailbox{}, FromTime: 0, ToTime: 30},
}, rules2.rules)
rules2.unsetRule(mailboxA)
rules2.unsetRule(mailboxC)
rules3 := loadRules(path, ruleID)
r.Equal(t, map[string]*Rule{
mailboxA.Hash(): {Active: false, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB, mailboxC}, FromTime: 0, ToTime: 0},
mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 10, ToTime: 20},
mailboxC.Hash(): {Active: false, SourceMailbox: mailboxC, TargetMailboxes: []Mailbox{}, FromTime: 0, ToTime: 30},
}, rules3.rules)
}
func TestSetGlobalTimeLimit(t *testing.T) {
path, err := ioutil.TempDir("", "rules")
r.NoError(t, err)
defer os.RemoveAll(path) //nolint[errcheck]
rules := loadRules(path, "rule")
mailboxA := Mailbox{Name: "One"}
mailboxB := Mailbox{Name: "Two"}
r.NoError(t, rules.setRule(mailboxA, []Mailbox{}, 10, 20))
r.NoError(t, rules.setRule(mailboxB, []Mailbox{}, 0, 0))
rules.setGlobalTimeLimit(30, 40)
rules.propagateGlobalTime()
r.Equal(t, map[string]*Rule{
mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{}, FromTime: 10, ToTime: 20},
mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{}, FromTime: 30, ToTime: 40},
}, rules.rules)
}
func TestSetDefaultRules(t *testing.T) {
path, err := ioutil.TempDir("", "rules")
r.NoError(t, err)
defer os.RemoveAll(path) //nolint[errcheck]
rules := loadRules(path, "rule")
mailbox1 := Mailbox{Name: "One"} // Set manually, default will not override it.
mailbox2 := Mailbox{Name: "Two"} // Matched by `targetMailboxes`.
mailbox3 := Mailbox{Name: "Three"} // Matched by `defaultCallback`, not included in `targetMailboxes`.
mailbox4 := Mailbox{Name: "Four"} // Matched by nothing, will not be active.
mailbox5 := Mailbox{Name: "Spam", ID: pmapi.SpamLabel} // Spam is inactive by default (ID found in source).
mailbox6a := Mailbox{Name: "Draft"} // Draft is inactive by default (ID found in target, mailbox6b).
mailbox6b := Mailbox{Name: "Draft", ID: pmapi.DraftLabel}
sourceMailboxes := []Mailbox{mailbox1, mailbox2, mailbox3, mailbox4, mailbox5, mailbox6a}
targetMailboxes := []Mailbox{mailbox1, mailbox2, mailbox6b}
r.NoError(t, rules.setRule(mailbox1, []Mailbox{mailbox3}, 0, 0))
defaultCallback := func(mailbox Mailbox) []Mailbox {
if mailbox.Name == "Three" {
return []Mailbox{mailbox3}
}
return []Mailbox{}
}
rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback)
r.Equal(t, map[string]*Rule{
mailbox1.Hash(): {Active: true, SourceMailbox: mailbox1, TargetMailboxes: []Mailbox{mailbox3}},
mailbox2.Hash(): {Active: true, SourceMailbox: mailbox2, TargetMailboxes: []Mailbox{mailbox2}},
mailbox3.Hash(): {Active: true, SourceMailbox: mailbox3, TargetMailboxes: []Mailbox{mailbox3}},
mailbox4.Hash(): {Active: false, SourceMailbox: mailbox4, TargetMailboxes: []Mailbox{}},
mailbox5.Hash(): {Active: false, SourceMailbox: mailbox5, TargetMailboxes: []Mailbox{}},
mailbox6a.Hash(): {Active: false, SourceMailbox: mailbox6a, TargetMailboxes: []Mailbox{mailbox6b}},
}, rules.rules)
}
func TestSetDefaultRulesDeactivateMissing(t *testing.T) {
path, err := ioutil.TempDir("", "rules")
r.NoError(t, err)
defer os.RemoveAll(path) //nolint[errcheck]
rules := loadRules(path, "rule")
mailboxA := Mailbox{ID: "1", Name: "One", Color: "", IsExclusive: true}
mailboxB := Mailbox{ID: "2", Name: "Two", Color: "", IsExclusive: true}
r.NoError(t, rules.setRule(mailboxA, []Mailbox{mailboxB}, 0, 0))
r.NoError(t, rules.setRule(mailboxB, []Mailbox{mailboxB}, 0, 0))
sourceMailboxes := []Mailbox{mailboxA}
targetMailboxes := []Mailbox{mailboxA, mailboxB}
defaultCallback := func(mailbox Mailbox) (mailboxes []Mailbox) {
return
}
rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback)
r.Equal(t, map[string]*Rule{
mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 0, ToTime: 0},
}, rules.rules)
}
func TestIsTimeInRange(t *testing.T) {
tests := []struct {
rule Rule
time int64
want bool
}{
{generateTimeRule(0, 0), 0, true},
{generateTimeRule(0, 0), 10, true},
{generateTimeRule(0, 15), 10, true},
{generateTimeRule(5, 15), 10, true},
{generateTimeRule(0, 5), 10, false},
{generateTimeRule(5, 7), 10, false},
{generateTimeRule(15, 30), 10, false},
{generateTimeRule(15, 0), 10, false},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v / %d", tc.rule, tc.time), func(t *testing.T) {
got := tc.rule.isTimeInRange(tc.time)
r.Equal(t, tc.want, got)
})
}
}
func TestHasTimeLimit(t *testing.T) {
tests := []struct {
rule Rule
want bool
}{
{generateTimeRule(0, 0), false},
{generateTimeRule(0, 1), true},
{generateTimeRule(1, 2), true},
{generateTimeRule(1, 0), true},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.rule), func(t *testing.T) {
r.Equal(t, tc.want, tc.rule.HasTimeLimit())
})
}
}
func generateTimeRule(from, to int64) Rule {
return Rule{
SourceMailbox: Mailbox{},
TargetMailboxes: []Mailbox{},
FromTime: from,
ToTime: to,
}
}
func TestOrderRules(t *testing.T) {
wantMailboxOrder := []Mailbox{
{Name: "Inbox", IsExclusive: true},
{Name: "Drafts", IsExclusive: true},
{Name: "Sent", IsExclusive: true},
{Name: "Starred", IsExclusive: true},
{Name: "Archive", IsExclusive: true},
{Name: "Spam", IsExclusive: true},
{Name: "All Mail", IsExclusive: true},
{Name: "Folder A", IsExclusive: true},
{Name: "Folder B", IsExclusive: true},
{Name: "Folder C", IsExclusive: true},
{Name: "Label A", IsExclusive: false},
{Name: "Label B", IsExclusive: false},
{Name: "Label C", IsExclusive: false},
}
wantMailboxNames := []string{}
rules := map[string]*Rule{}
for _, mailbox := range wantMailboxOrder {
wantMailboxNames = append(wantMailboxNames, mailbox.Name)
rules[mailbox.Hash()] = &Rule{
SourceMailbox: mailbox,
}
}
transferRules := transferRules{
rules: rules,
}
gotMailboxNames := []string{}
for _, rule := range transferRules.getSortedRules() {
gotMailboxNames = append(gotMailboxNames, rule.SourceMailbox.Name)
}
r.Equal(t, wantMailboxNames, gotMailboxNames)
}

4
internal/transfer/testdata/eml/Foo/msg.eml vendored
View File

@ -1,4 +0,0 @@
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
hello

4
internal/transfer/testdata/eml/Inbox/msg.eml vendored
View File

@ -1,4 +0,0 @@
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
hello

4
internal/transfer/testdata/emlmbox/Foo/msg.eml vendored
View File

@ -1,4 +0,0 @@
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
hello

5
internal/transfer/testdata/emlmbox/Inbox.mbox vendored
View File

@ -1,5 +0,0 @@
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
hello

62
internal/transfer/testdata/keyring_userKey vendored
View File

@ -1,62 +0,0 @@
-----BEGIN PGP PRIVATE KEY BLOCK-----
Version: OpenPGP.js v4.4.5
Comment: testpassphrase
xcLYBFzGzhEBCADBxfqTFMqfQzT77A5tuuhPFwPq8dfC2evs8u1OvTqFbztY
5FOuSxzduyeDqQ1Fx6dKEOKgcYE8t1Uh4VSS7z6bTdY8j9yrL81kCVB46sE1
OzStzyx/5l7OdH/pM4F+aKslnLvqlw0UeJr+UNizVtOCEUaNfVjPK3cc1ocx
v+36K4RnnyfEtjUW9gDZbhgaF02G5ILHmWmbgM7I+77gCd2wI0EdY9s/JZQ+
VmkMFqoMdY9PyBchoOIPUkkGQi1SaF4IEzMaAUSbnCYkHHY/SbfDTcR46VGq
cXlkB1rq5xskaUQ9r+giCC/K4pc7bBkI1lQ7ADVuWvdrWnWapK0FO6CfABEB
AAEAB/0YPhPJ0phA/EWviN+16bmGVOZNaVapjt2zMMybWmrtEQv3OeWgO3nP
4cohRi/zaCBCphcm+dxbLhftW7AFi/9PVcR09436MB+oTCQFugpUWw+4TmA5
BidxTpDxf4X2vH3rquQLBufWL6U7JlPeKAGL1xZ2aCq0DIeOk5D+xTjZizV2
GIyQRVCLWb+LfDmvvcp3Y94X60KXdBAMuS1ZMKcY3Sl8VAXNB4KQsC/kByzf
6FCB097XZRYV7lvJJQ7+6Wisb3yVi8sEQx2sFm5fAp+0qi3a6zRTEp49r6Hr
gyWViH5zOOpA7DcNwx1Bwhi7GG0tak6EUnnKUNLfOupglcphBADmpXCgT4nc
uSBYTiZSVcB/ICCkTxVsHL1WcXtPK2Ikzussx2n9kb0rapvuC0YLipX9lUkQ
fyeC3jQJeCyN79AkDGkOfWaESueT2hM0Po+RwDgMibKn6yJ1zebz4Lc2J3C9
oVFcAnql+9KyGsAPn03fyQzDnvhNnJvHJi4Hx8AWoQQA1xLoXeVBjRi0IjjU
E6Mqaq5RLEog4kXRp86VSSEGHBwyIYnDiM//gjseo/CXuVyHwL7UXitp8s1B
D1uE3APrhqUS66fD5pkF+z+RcSqiIv7I76NJ24Cdg38L6seGSjOHrq7/dEeG
K6WqfQUCEjta3yNSg7pXb2wn2WZqKIK+rz8EALZRuMXeql/FtO3Cjb0sv7oT
9dLP4cn1bskGRJ+Vok9lfCERbfXGccoAk3V+qSfpHgKxsebkRbUhf+trOGnw
tW+kBWo/5hYGQuN+A9JogSJViT+nuZyE+x1/rKswDFmlMSdf2GIDARWIV0gc
b1yOEwUmNBSthPcnFXvBr4BG3XTtNPTNLSJhcm9uMjEtM0BzYWRlbWJlLm9y
ZyIgPGFyb24yMS0zQHNhZGVtYmUub3JnPsLAdQQQAQgAHwUCXMbOEQYLCQcI
AwIEFQgKAgMWAgECGQECGwMCHgEACgkQZ/B4v2b2xB6XUgf/dHGRHimyMR78
QYbEm2cuaEvOtq4a+J6Zv3P4VOWAbvkGWS9LDKSvVi60vq4oYOmF54HgPzur
nA4OtZDf0HKwQK45VZ7CYD693o70jkKPrAAJG3yTsbesfiS7RbFyGKzKJ7EL
nsUIJkfgm/SlKmXU/u8MOBO5Wg7/TcsS33sRWHl90j+9jbhqdl92R+vY/CwC
ieFkQA7/TDv1u+NAalH+Lpkd8AIuEcki+TAogZ7oi/SnofwnoB7BxRm+mIkp
ZZhIDSCaPOzLG8CSZ81d3HVHhqbf8dh0DFKFoUYyKdbOqIkNWWASf+c/ZEme
IWcekY8hqwf/raZ56tGM/bRwYPcotMfC1wRcxs4RAQgAsMb5/ELWmrfPy3ba
5qif+RXhGSbjitATNgHpoPUHrfTC7cn4JWHqehoXLAQpFAoKd+O/ZNpZozK9
ilpqGUx05yMw06jNQEhYIbgIF4wzPpz02Lp6YeMwdF5LF+Rw83PHdHrA/wRV
/QjL04+kZnN+G5HmzMlhFY+oZSpL+Gp1bTXgtAVDkhCnMB5tP2VwULMGyJ+X
vRYxwTK2CrLjIVZv5n1VYY+caCowU6j/XFqvlCJj+G5oV+UhFOWffaMRXhOh
a64RrhqT1Np7wCLvLMP2wpys9xlMcLQJLqDNxqOTp504V7dm67ncC0fKUsT4
m4oTktnxKPd6MU+4VYveaLCquwARAQABAAf4u9s7gpGErs1USxmDO9TlyGZK
aBlri8nMf3s+hOJCOo3cRaRHJBfdY6pu/baG6H6JTsWzeY4MHwr6N+dhVIEh
FPMa9EZAjagyc4GugxWGiMVTfU+2AEfdrdynhQKMgXSctnnNCdkRuX0nwqb3
nlupm1hsz2ze4+Wg0BKSLS0FQdoUbITdJUR69OHr4dNJVHWYI0JSBx4SdhV3
y9163dDvmc+lW9AEaD53vyZWfzCHZxsR/gI32VmT0z5gn1t8w9AOdXo2lA1H
bf7wh4/qCyujGu64ToZtiEny/GCyM6PofLtiZuJNLw3s/y+B2tKv22aTJ760
+Gib1xB9WcWjKyrxBADoeCyq+nHGrl0CwOkmjanlFymgo7mnBOXuiFOvGrKk
M1meMU1TI4TEBWkVnDVMcSejgjAf/bX1dtouba1tMAMu7DlaV/0EwbSADRel
RSqEbIzIOys+y9TY/BMI/uCKNyEKHvu1KUXADb+CBpdBpCfMBWDANFlo9xLz
Ajcmu2dyawQAwquwC0VXQcvzfs+Hd5au5XvHdm1KidOiAdu6PH1SrOgenIN4
lkEjHrJD9jmloO2/GVcxDBB2pmf0B4HEg7DuY9LXBrksP5eSbbRc5+UH1HUv
u82AqQnfNKTd/jae+lLwaOS++ohtwMkkD6W0LdWnHPjyyXg4Oi9zPID3asRu
3PED/3CYyjl6S8GTMY4FNH7Yxu9+NV2xpKE92Hf8K/hnYlmSSVKDCEeOJtLt
BkkcSqY6liCNSMmJdVyAF2GrR+zmDac7UQRssf57oOWtSsGozt0aqJXuspMT
6aB+P1UhZ8Ly9rWZNiJ0jwyfnQNOLCYDaqjFmiSpqrNnJ2Q1Xge3+k80P9DC
wF8EGAEIAAkFAlzGzhECGwwACgkQZ/B4v2b2xB5wlwgAjZA1zdv5irFjyWVo
4/itONtyO1NbdpyYpcct7vD0oV+a4wahQP0J3Kk1GhZ5tvAoZF/jakQQOM5o
GjUYpXAGnr09Mv9EiQ2pDwXc2yq0WfXnGxNrpzOqdtV+IqY9NYkl55Tme7x+
WRvrkPSUeUsyEGvxwR1stdv8eg9jUmxdl8Io3PYoFJJlrM/6aXeC1r3KOj7q
XAnR0XHJ+QBSNKCWLlQv5hui9BKfcLiVKFK/dNhs82nRyhPr4sWFw6MTqdAK
4zkn7l0jmy6Evi1AiiGPiHPnxeNErnofOIEh4REQj00deZADHrixTLtx2FuR
uaSC3IcBmBsj1fNb4eYXElILjQ==
=fMOl
-----END PGP PRIVATE KEY BLOCK-----

16
internal/transfer/testdata/mbox-applemail/All Mail.mbox/mbox vendored
View File

@ -1,16 +0,0 @@
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
Subject: Test 1
X-Gmail-Labels: Foo,Bar
hello
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
Subject: Test 2
X-Gmail-Labels: Foo
hello

0
internal/transfer/testdata/mbox-applemail/Inbox.mbox/.keep vendored
View File

16
internal/transfer/testdata/mbox/All Mail.mbox vendored
View File

@ -1,16 +0,0 @@
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
Subject: Test 1
X-Gmail-Labels: Foo,Bar
hello
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
Subject: Test 2
X-Gmail-Labels: Foo
hello

5
internal/transfer/testdata/mbox/Foo.mbox vendored
View File

@ -1,5 +0,0 @@
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
hello

5
internal/transfer/testdata/mbox/Inbox.mbox vendored
View File

@ -1,5 +0,0 @@
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
hello

80
internal/transfer/timeit.go
View File

@ -1,80 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"sync"
"time"
)
type timeIt struct {
lock sync.Locker
name string
groups map[string]int64
ongoing map[string]time.Time
}
func newTimeIt(name string) *timeIt {
return &timeIt{
lock: &sync.Mutex{},
name: name,
groups: map[string]int64{},
ongoing: map[string]time.Time{},
}
}
func (t *timeIt) clear() {
t.lock.Lock()
defer t.lock.Unlock()
t.groups = map[string]int64{}
t.ongoing = map[string]time.Time{}
}
func (t *timeIt) start(group, id string) {
t.lock.Lock()
defer t.lock.Unlock()
t.ongoing[group+"/"+id] = time.Now()
}
func (t *timeIt) stop(group, id string) {
endTime := time.Now()
t.lock.Lock()
defer t.lock.Unlock()
startTime, ok := t.ongoing[group+"/"+id]
if !ok {
log.WithField("group", group).WithField("id", id).Error("Stop called before start")
return
}
delete(t.ongoing, group+"/"+id)
diff := endTime.Sub(startTime).Milliseconds()
t.groups[group] += diff
}
func (t *timeIt) logResults() {
t.lock.Lock()
defer t.lock.Unlock()
// Print also ongoing to be sure that nothing was left out.
// Basically ongoing should be empty.
log.WithField("name", t.name).WithField("result", t.groups).WithField("ongoing", t.ongoing).Debug("Time measurement")
}

214
internal/transfer/transfer.go
View File

@ -1,214 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
// Package transfer provides tools to export messages from one provider and
// import them to another provider. Provider can be EML, MBOX, IMAP or PMAPI.
package transfer
import (
"crypto/sha256"
"fmt"
"github.com/sirupsen/logrus"
)
var log = logrus.WithField("pkg", "transfer") //nolint[gochecknoglobals]
// Transfer is facade on top of import rules, progress manager and source
// and target providers. This is the main object which should be used.
type Transfer struct {
panicHandler PanicHandler
metrics MetricsManager
id string
logDir string
rules transferRules
source SourceProvider
target TargetProvider
rulesCache []*Rule
sourceMboxCache []Mailbox
targetMboxCache []Mailbox
}
// New creates Transfer for specific source and target. Usage:
//
// source := transfer.NewEMLProvider(...)
// target := transfer.NewPMAPIProvider(...)
// transfer.New(source, target, ...)
func New(panicHandler PanicHandler, metrics MetricsManager, logDir, rulesDir string, source SourceProvider, target TargetProvider) (*Transfer, error) {
transferID := fmt.Sprintf("%x", sha256.Sum256([]byte(source.ID()+"-"+target.ID())))
rules := loadRules(rulesDir, transferID)
transfer := &Transfer{
panicHandler: panicHandler,
metrics: metrics,
id: transferID,
logDir: logDir,
rules: rules,
source: source,
target: target,
}
if err := transfer.setDefaultRules(); err != nil {
return nil, err
}
metrics.Load(len(transfer.sourceMboxCache))
return transfer, nil
}
// SetDefaultRules sets missing rules for source mailboxes with matching
// target mailboxes. In case no matching mailbox is found, `defaultCallback`
// with a source mailbox as a parameter is used.
func (t *Transfer) setDefaultRules() error {
sourceMailboxes, err := t.SourceMailboxes()
if err != nil {
return err
}
targetMailboxes, err := t.TargetMailboxes()
if err != nil {
return err
}
defaultCallback := func(sourceMailbox Mailbox) []Mailbox {
return t.target.DefaultMailboxes(sourceMailbox)
}
t.rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback)
return nil
}
// SetSkipEncryptedMessages sets whether message which cannot be decrypted
// should be imported/exported or skipped.
func (t *Transfer) SetSkipEncryptedMessages(skip bool) {
t.rules.setSkipEncryptedMessages(skip)
}
// SetGlobalMailbox sets mailbox that is applied to every message in
// the import phase.
func (t *Transfer) SetGlobalMailbox(mailbox *Mailbox) {
t.rules.setGlobalMailbox(mailbox)
}
// SetGlobalTimeLimit sets time limit that is applied to rules without any
// specified time limit.
func (t *Transfer) SetGlobalTimeLimit(fromTime, toTime int64) {
t.rules.setGlobalTimeLimit(fromTime, toTime)
}
// SetRule sets sourceMailbox for transfer.
func (t *Transfer) SetRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error {
t.rulesCache = nil
return t.rules.setRule(sourceMailbox, targetMailboxes, fromTime, toTime)
}
// UnsetRule unsets sourceMailbox from transfer.
func (t *Transfer) UnsetRule(sourceMailbox Mailbox) {
t.rulesCache = nil
t.rules.unsetRule(sourceMailbox)
}
// ResetRules unsets all rules.
func (t *Transfer) ResetRules() {
t.rulesCache = nil
t.rules.reset()
}
// GetRule returns rule for given mailbox.
func (t *Transfer) GetRule(sourceMailbox Mailbox) *Rule {
return t.rules.getRule(sourceMailbox)
}
// GetRules returns all set transfer rules.
func (t *Transfer) GetRules() []*Rule {
if t.rulesCache == nil {
t.rulesCache = t.rules.getSortedRules()
}
return t.rulesCache
}
// SourceMailboxes returns mailboxes available at source side.
func (t *Transfer) SourceMailboxes() (m []Mailbox, err error) {
if t.sourceMboxCache == nil {
t.sourceMboxCache, err = t.source.Mailboxes(false, true)
}
return t.sourceMboxCache, err
}
// TargetMailboxes returns mailboxes available at target side.
func (t *Transfer) TargetMailboxes() (m []Mailbox, err error) {
if t.targetMboxCache == nil {
t.targetMboxCache, err = t.target.Mailboxes(true, false)
}
return t.targetMboxCache, err
}
// CreateTargetMailbox creates mailbox in target provider.
func (t *Transfer) CreateTargetMailbox(mailbox Mailbox) (Mailbox, error) {
t.targetMboxCache = nil
return t.target.CreateMailbox(mailbox)
}
// ChangeTarget changes the target. It is safe to change target for export,
// must not be changed for import. Do not set after you started transfer.
func (t *Transfer) ChangeTarget(target TargetProvider) {
t.targetMboxCache = nil
t.target = target
}
// Start starts the transfer from source to target.
func (t *Transfer) Start() *Progress {
log.Debug("Transfer started")
t.rules.save()
t.rules.propagateGlobalTime()
t.metrics.Start()
log := log.WithField("id", t.id)
reportFile := newFileReport(t.logDir, t.id)
progress := newProgress(log, reportFile)
// Small queue to prevent having idle source while target is blocked.
// E.g., when upload to PM is in progress, we can in meantime download
// the next batch from remote IMAP server.
ch := make(chan Message, 10)
go func() {
defer t.panicHandler.HandlePanic()
t.source.TransferTo(t.rules, &progress, ch)
close(ch)
}()
go func() {
defer t.panicHandler.HandlePanic()
t.target.TransferFrom(t.rules, &progress, ch)
progress.finish()
if progress.isStopped {
if progress.fatalError != nil {
t.metrics.Fail()
} else {
t.metrics.Cancel()
}
} else {
t.metrics.Complete()
}
}()
return &progress
}

71
internal/transfer/transfer_test.go
View File

@ -1,71 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"io/ioutil"
"testing"
"github.com/ProtonMail/gopenpgp/v2/crypto"
transfermocks "github.com/ProtonMail/proton-bridge/internal/transfer/mocks"
pmapimocks "github.com/ProtonMail/proton-bridge/pkg/pmapi/mocks"
gomock "github.com/golang/mock/gomock"
)
type mocks struct {
t *testing.T
ctrl *gomock.Controller
panicHandler *transfermocks.MockPanicHandler
imapClientProvider *transfermocks.MockIMAPClientProvider
pmapiClient *pmapimocks.MockClient
keyring *crypto.KeyRing
}
func initMocks(t *testing.T) mocks {
mockCtrl := gomock.NewController(t)
m := mocks{
t: t,
ctrl: mockCtrl,
panicHandler: transfermocks.NewMockPanicHandler(mockCtrl),
imapClientProvider: transfermocks.NewMockIMAPClientProvider(mockCtrl),
pmapiClient: pmapimocks.NewMockClient(mockCtrl),
keyring: newTestKeyring(),
}
return m
}
func newTestKeyring() *crypto.KeyRing {
data, err := ioutil.ReadFile("testdata/keyring_userKey")
if err != nil {
panic(err)
}
key, err := crypto.NewKeyFromArmored(string(data))
if err != nil {
panic(err)
}
userKey, err := crypto.NewKeyRing(key)
if err != nil {
panic(err)
}
return userKey
}

30
internal/transfer/types.go
View File

@ -1,30 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
type PanicHandler interface {
HandlePanic()
}
type MetricsManager interface {
Load(int)
Start()
Complete()
Cancel()
Fail()
}

186
internal/transfer/utils.go
View File

@ -1,186 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"bufio"
"bytes"
"io/ioutil"
"net/mail"
"net/textproto"
"path/filepath"
"runtime"
"sort"
"strings"
"github.com/ProtonMail/go-rfc5322"
"github.com/pkg/errors"
)
// getFolderNames collects all folder names under `root`.
// Folder names will be without a path.
func getFolderNames(root string) ([]string, error) {
return getFolderNamesWithFileSuffix(root, "")
}
// getFolderNamesWithFileSuffix collects all folder names under `root`, which
// contains some file with a give `fileSuffix`. Names will be without a path.
func getFolderNamesWithFileSuffix(root, fileSuffix string) ([]string, error) {
folders := []string{}
files, err := ioutil.ReadDir(root)
if err != nil {
return nil, err
}
hasFileWithSuffix := fileSuffix == ""
for _, file := range files {
if file.IsDir() {
subfolders, err := getFolderNamesWithFileSuffix(filepath.Join(root, file.Name()), fileSuffix)
if err != nil {
return nil, err
}
for _, subfolder := range subfolders {
match := false
for _, folder := range folders {
if folder == subfolder {
match = true
break
}
}
if !match {
folders = append(folders, subfolder)
}
}
} else if fileSuffix == "" || strings.HasSuffix(file.Name(), fileSuffix) {
hasFileWithSuffix = true
}
}
if hasFileWithSuffix {
folders = append(folders, filepath.Base(root))
}
sort.Strings(folders)
return folders, nil
}
// getFilePathsWithSuffix collects all file names with `suffix` under `root`.
// File names will be with relative path based to `root`.
func getFilePathsWithSuffix(root, suffix string) ([]string, error) {
fileNames, err := getFilePathsWithSuffixInner("", root, suffix, false)
if err != nil {
return nil, err
}
sort.Strings(fileNames)
return fileNames, err
}
// getAllPathsWithSuffix is the same as getFilePathsWithSuffix but includes
// also directories.
func getAllPathsWithSuffix(root, suffix string) ([]string, error) {
fileNames, err := getFilePathsWithSuffixInner("", root, suffix, true)
if err != nil {
return nil, err
}
sort.Strings(fileNames)
return fileNames, err
}
func getFilePathsWithSuffixInner(prefix, root, suffix string, includeDir bool) ([]string, error) {
fileNames := []string{}
files, err := ioutil.ReadDir(root)
if err != nil {
return nil, err
}
for _, file := range files {
if !file.IsDir() {
if strings.HasSuffix(file.Name(), suffix) {
fileNames = append(fileNames, filepath.Join(prefix, file.Name()))
}
} else {
if includeDir && strings.HasSuffix(file.Name(), suffix) {
fileNames = append(fileNames, filepath.Join(prefix, file.Name()))
}
subfolderFileNames, err := getFilePathsWithSuffixInner(
filepath.Join(prefix, file.Name()),
filepath.Join(root, file.Name()),
suffix,
includeDir,
)
if err != nil {
return nil, err
}
fileNames = append(fileNames, subfolderFileNames...)
}
}
return fileNames, nil
}
// getMessageTime returns time of the message specified in the message header.
func getMessageTime(body []byte) (int64, error) {
hdr, err := getMessageHeader(body)
if err != nil {
return 0, err
}
t, err := rfc5322.ParseDateTime(hdr.Get("Date"))
if err != nil {
return 0, err
}
if t.IsZero() {
return 0, nil
}
return t.Unix(), nil
}
// getMessageHeader returns headers of the message body.
func getMessageHeader(body []byte) (mail.Header, error) {
tpr := textproto.NewReader(bufio.NewReader(bytes.NewBuffer(body)))
header, err := tpr.ReadMIMEHeader()
if err != nil {
return nil, errors.Wrap(err, "failed to read headers")
}
return mail.Header(header), nil
}
// sanitizeFileName replaces problematic special characters with underscore.
func sanitizeFileName(fileName string) string {
if len(fileName) == 0 {
return fileName
}
if runtime.GOOS != "windows" && (fileName[0] == '-' || fileName[0] == '.') { //nolint[goconst]
fileName = "_" + fileName[1:]
}
return strings.Map(func(r rune) rune {
switch r {
case '\\', '/', ':', '*', '?', '"', '<', '>', '|':
return '_'
case '[', ']', '(', ')', '{', '}', '^', '#', '%', '&', '!', '@', '+', '=', '\'', '~':
if runtime.GOOS != "windows" {
return '_'
}
}
return r
}, fileName)
}

225
internal/transfer/utils_test.go
View File

@ -1,225 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
"testing"
r "github.com/stretchr/testify/require"
)
func TestGetFolderNames(t *testing.T) {
root, clean := createTestingFolderStructure(t)
defer clean()
tests := []struct {
suffix string
wantNames []string
}{
{
"",
[]string{
"bar",
"bar.mbox",
"baz",
filepath.Base(root),
"foo",
"qwerty",
"test",
},
},
{
".eml",
[]string{
"bar",
"baz",
filepath.Base(root),
"foo",
},
},
{
".txt",
[]string{
filepath.Base(root),
},
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.suffix, func(t *testing.T) {
names, err := getFolderNamesWithFileSuffix(root, tc.suffix)
r.NoError(t, err)
r.Equal(t, tc.wantNames, names)
})
}
}
func TestGetFilePathsWithSuffix(t *testing.T) {
root, clean := createTestingFolderStructure(t)
defer clean()
tests := []struct {
suffix string
wantPaths []string
}{
{
".eml",
[]string{
"foo/bar/baz/msg1.eml",
"foo/bar/baz/msg2.eml",
"foo/bar/baz/msg3.eml",
"foo/bar/msg4.eml",
"foo/bar/msg5.eml",
"foo/baz/msg6.eml",
"foo/msg7.eml",
"msg10.eml",
"test/foo/msg8.eml",
"test/foo/msg9.eml",
},
},
{
".mbox",
[]string{
"bar.mbox",
"foo.mbox",
},
},
{
".txt",
[]string{
"info.txt",
},
},
{
".hello",
[]string{},
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.suffix, func(t *testing.T) {
paths, err := getAllPathsWithSuffix(root, tc.suffix)
r.NoError(t, err)
r.Equal(t, tc.wantPaths, paths)
})
}
}
func createTestingFolderStructure(t *testing.T) (string, func()) {
root, err := ioutil.TempDir("", "folderstructure")
r.NoError(t, err)
for _, path := range []string{
"foo/bar/baz",
"foo/baz",
"test/foo",
"qwerty",
"bar.mbox",
} {
err = os.MkdirAll(filepath.Join(root, path), os.ModePerm)
r.NoError(t, err)
}
for _, path := range []string{
"foo/bar/baz/msg1.eml",
"foo/bar/baz/msg2.eml",
"foo/bar/baz/msg3.eml",
"foo/bar/msg4.eml",
"foo/bar/msg5.eml",
"foo/baz/msg6.eml",
"foo/msg7.eml",
"test/foo/msg8.eml",
"test/foo/msg9.eml",
"msg10.eml",
"info.txt",
"foo.mbox",
"bar.mbox/mbox", // Apple Mail mbox export format.
} {
f, err := os.Create(filepath.Join(root, path))
r.NoError(t, err)
err = f.Close()
r.NoError(t, err)
}
return root, func() {
_ = os.RemoveAll(root)
}
}
func TestGetMessageTime(t *testing.T) {
tests := []struct {
body string
wantTime int64
wantErr string
}{
{"", 0, "failed to read headers: EOF"},
{"Subject: hello\n\n", 0, ""},
{"Date: Thu, 23 Apr 2020 04:52:44 +0000\n\n", 1587617564, ""},
}
for _, tc := range tests {
tc := tc
t.Run(tc.body, func(t *testing.T) {
time, err := getMessageTime([]byte(tc.body))
if tc.wantErr == "" {
r.NoError(t, err)
} else {
r.EqualError(t, err, tc.wantErr)
}
r.Equal(t, tc.wantTime, time)
})
}
}
func TestGetMessageHeader(t *testing.T) {
body := `Subject: Hello
From: user@example.com
Body
`
header, err := getMessageHeader([]byte(body))
r.NoError(t, err)
r.Equal(t, header.Get("subject"), "Hello")
r.Equal(t, header.Get("from"), "user@example.com")
}
func TestSanitizeFileName(t *testing.T) {
tests := map[string]string{
"hello": "hello",
"a\\b/c:*?d\"<>|e": "a_b_c___d____e",
}
if runtime.GOOS == "darwin" || runtime.GOOS == "linux" {
tests[".hello"] = "_hello"
tests["-hello"] = "_hello"
}
if runtime.GOOS == "windows" {
tests["[hello]&@=~~"] = "_hello______"
}
for path, wantPath := range tests {
path := path
wantPath := wantPath
t.Run(path, func(t *testing.T) {
gotPath := sanitizeFileName(path)
r.Equal(t, wantPath, gotPath)
})
}
}

18
internal/users/user.go
View File

@ -50,8 +50,6 @@ type User struct {
creds *credentials.Credentials
lock sync.RWMutex
useOnlyActiveAddresses bool
}
// newUser creates a new user.
@ -62,7 +60,6 @@ func newUser(
eventListener listener.Listener,
credStorer CredentialsStorer,
storeFactory StoreMaker,
useOnlyActiveAddresses bool,
) (*User, *credentials.Credentials, error) {
log := log.WithField("user", userID)
@ -74,14 +71,13 @@ func newUser(
}
return &User{
log: log,
panicHandler: panicHandler,
listener: eventListener,
credStorer: credStorer,
storeFactory: storeFactory,
userID: userID,
creds: creds,
useOnlyActiveAddresses: useOnlyActiveAddresses,
log: log,
panicHandler: panicHandler,
listener: eventListener,
credStorer: credStorer,
storeFactory: storeFactory,
userID: userID,
creds: creds,
}, creds, nil
}

2
internal/users/user_credentials_test.go
View File

@ -172,7 +172,7 @@ func TestCheckBridgeLoginLoggedOut(t *testing.T) {
m.eventListener.EXPECT().Emit(events.LogoutEvent, "user"),
)
user, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker, false)
user, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker)
r.NoError(t, err)
err = user.connect(m.pmapiClient, testCredentialsDisconnected)

4
internal/users/user_new_test.go
View File

@ -33,7 +33,7 @@ func TestNewUserNoCredentialsStore(t *testing.T) {
m.credentialsStore.EXPECT().Get("user").Return(nil, errors.New("fail"))
_, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker, false)
_, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker)
r.Error(t, err)
}
@ -71,7 +71,7 @@ func TestNewUser(t *testing.T) {
}
func checkNewUserHasCredentials(m mocks, wantErr string, wantCreds *credentials.Credentials) {
user, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker, false)
user, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker)
r.NoError(m.t, err)
defer cleanUpUserData(user)

2
internal/users/user_test.go
View File

@ -27,7 +27,7 @@ func testNewUser(m mocks) *User {
mockInitConnectedUser(m)
mockEventLoopNoAction(m)
user, creds, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker, false)
user, creds, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker)
r.NoError(m.t, err)
err = user.connect(m.pmapiClient, creds)

36
internal/users/users.go
View File

@ -58,12 +58,6 @@ type Users struct {
// People are used to that and so we preserve that ordering here.
users []*User
// useOnlyActiveAddresses determines whether credentials keeps only active
// addresses or all of them. Each usage has to be consisteng, e.g., once
// user is added, it saves address list to credentials and next time loads
// as is, without requesting server again.
useOnlyActiveAddresses bool
lock sync.RWMutex
}
@ -74,19 +68,17 @@ func New(
clientManager pmapi.Manager,
credStorer CredentialsStorer,
storeFactory StoreMaker,
useOnlyActiveAddresses bool,
) *Users {
log.Trace("Creating new users")
u := &Users{
locations: locations,
panicHandler: panicHandler,
events: eventListener,
clientManager: clientManager,
credStorer: credStorer,
storeFactory: storeFactory,
useOnlyActiveAddresses: useOnlyActiveAddresses,
lock: sync.RWMutex{},
locations: locations,
panicHandler: panicHandler,
events: eventListener,
clientManager: clientManager,
credStorer: credStorer,
storeFactory: storeFactory,
lock: sync.RWMutex{},
}
go func() {
@ -135,7 +127,7 @@ func (u *Users) loadUsersFromCredentialsStore() error {
for _, userID := range userIDs {
l := log.WithField("user", userID)
user, creds, err := newUser(u.panicHandler, userID, u.events, u.credStorer, u.storeFactory, u.useOnlyActiveAddresses)
user, creds, err := newUser(u.panicHandler, userID, u.events, u.credStorer, u.storeFactory)
if err != nil {
l.WithError(err).Warn("Could not create user, skipping")
continue
@ -256,19 +248,11 @@ func (u *Users) addNewUser(client pmapi.Client, apiUser *pmapi.User, auth *pmapi
u.lock.Lock()
defer u.lock.Unlock()
var emails []string
if u.useOnlyActiveAddresses {
emails = client.Addresses().ActiveEmails()
} else {
emails = client.Addresses().AllEmails()
}
if _, err := u.credStorer.Add(apiUser.ID, apiUser.Name, auth.UID, auth.RefreshToken, passphrase, emails); err != nil {
if _, err := u.credStorer.Add(apiUser.ID, apiUser.Name, auth.UID, auth.RefreshToken, passphrase, client.Addresses().ActiveEmails()); err != nil {
return errors.Wrap(err, "failed to add user credentials to credentials store")
}
user, creds, err := newUser(u.panicHandler, apiUser.ID, u.events, u.credStorer, u.storeFactory, u.useOnlyActiveAddresses)
user, creds, err := newUser(u.panicHandler, apiUser.ID, u.events, u.credStorer, u.storeFactory)
if err != nil {
return errors.Wrap(err, "failed to create new user")
}

2
internal/users/users_test.go
View File

@ -223,7 +223,7 @@ func testNewUsers(t *testing.T, m mocks) *Users { //nolint[unparam]
m.eventListener.EXPECT().ProvideChannel(events.UpgradeApplicationEvent)
m.eventListener.EXPECT().ProvideChannel(events.InternetOnEvent)
users := New(m.locator, m.PanicHandler, m.eventListener, m.clientManager, m.credentialsStore, m.storeMaker, true)
users := New(m.locator, m.PanicHandler, m.eventListener, m.clientManager, m.credentialsStore, m.storeMaker)
waitForEvents()

15
release-notes/ie_early.md
View File

@ -1,15 +0,0 @@
## v1.3.0
- 2021-02-02
### New
- Introducing silent updates
- Improvements to message parsing
### Fixed
- Setting up flags to avoid messages misplacement
- Remove dependency on go-apple-mobileconfig
- Change to how attachment size is processed to avoid potential errors
- Linux font issues - Fedora specific

57
release-notes/ie_stable.md
View File

@ -1,57 +0,0 @@
## v1.3.3
- 2021-05-17
### Fixed
- Fixed potential security vulnerability related to rpath
- Improved parsing of embedded messages
## v1.3.1
- 2021-03-11
### New
- Reduce the number of import errors by supporting malformed undisclosed-recipient and better handling of overly long headers
- Improvements to how large attachments are processed
- New format of the release notes
### Fixed
- Linux font issues - Fedora specific
- Fixed rare message misplacement
- Ensure removal of the startup entry during uninstallation
- Update errors
## v1.2.2
- 2020-11-27
### New
- Improvements to the import from large mbox files with multiple labels
- Not allow to run multiple instances of the app or transfers at the same time
- Better handling and displaying of skipped messages
- Various enhancements of the import process related to parsing
- Cosmetic GUI changes
- Better error handling
### Fixed
- Linux font issues - Fedora specific
- App response to the user pausing and canceling import or export
- Upgrade errors
## v1.1.2
- 2020-09-23
### New
- Improving performance
- Speed up import by implementing parallel processing (parallel fetch, encrypt and upload of messages)
- Optimising the initial fetch of messages from external accounts
- Better message parsing
- Better handling of attachments and non-standard formatting
- Improved stability of the message parser
- Improved metrics
- Added persistent anonymous API cookies
### Fixed
- Fixed issues causing failing of import
- Import from mbox files with long lines
- Improvements to import from Yahoo accounts

25
test/Makefile
View File

@ -1,10 +1,9 @@
.PHONY: check-go check-godog install-godog test test-bridge test-ie test-live test-live-bridge test-live-ie test-stage test-debug test-live-debug bench
.PHONY: check-go check-godog install-godog test test-bridge test-live test-live-bridge test-stage test-debug test-live-debug bench
export GO111MODULE=on
export BRIDGE_VERSION:=1.8.12+integrationtests
export VERBOSITY?=fatal
export TEST_DATA=testdata
export TEST_APP?=bridge
# Tests do not run in parallel. This will overrule user settings.
MAKEFLAGS=-j1
@ -16,23 +15,17 @@ check-godog:
install-godog: check-go
go get github.com/cucumber/godog/cmd/godog@v0.12.1
test: test-bridge test-ie
test-bridge: FEATURES ?= features/bridge
test: test-bridge
test-bridge: FEATURES ?= features
test-bridge: check-godog
TEST_APP=bridge TEST_ENV=fake TEST_ACCOUNTS=accounts/fake.json godog --tags="~@ignore" $(FEATURES)
test-ie: FEATURES ?= features/ie
test-ie: check-godog
TEST_APP=ie TEST_ENV=fake TEST_ACCOUNTS=accounts/fake.json godog --tags="~@ignore" $(FEATURES)
TEST_ENV=fake TEST_ACCOUNTS=accounts/fake.json godog --tags="~@ignore" $(FEATURES)
# Doesn't work in parallel!
# Provide TEST_ACCOUNTS with your accounts.
test-live: test-live-bridge test-live-ie
test-live-bridge: FEATURES ?= features/bridge
test-live-bridge: FEATURES ?= features
test-live-bridge: check-godog
TEST_APP=bridge TEST_ENV=live godog --tags="~@ignore && ~@ignore-live" $(FEATURES)
test-live-ie: FEATURES ?= features/ie
test-live-ie: check-godog
TEST_APP=ie TEST_ENV=live godog --tags="~@ignore && ~@ignore-live" $(FEATURES)
TEST_ENV=live godog --tags="~@ignore && ~@ignore-live" $(FEATURES)
# Doesn't work in parallel!
# Provide TEST_ACCOUNTS with your accounts.
@ -47,12 +40,6 @@ test-debug:
test-live-debug:
TEST_ENV=live dlv test -- $(FEATURES)
test-ie-debug:
TEST_APP=ie TEST_ENV=fake TEST_ACCOUNTS=accounts/fake.json dlv test -- $(FEATURES)
test-live-ie-debug:
TEST_APP=ie TEST_ENV=live dlv test -- $(FEATURES)
# -run flag is not working anyway, but lets keep it there to note we really do not want to run tests.
# To properly benchmark sync/fetch, we need everything empty. For that is better to start everything
# again and safest way is to run only one loop per run.

9
test/bdd_test.go
View File

@ -19,7 +19,6 @@ package tests
import (
"context"
"os"
testContext "github.com/ProtonMail/proton-bridge/test/context"
"github.com/cucumber/godog"
@ -60,10 +59,6 @@ func ScenarioInitializer(s *godog.ScenarioContext) {
StoreChecksFeatureContext(s)
StoreSetupFeatureContext(s)
TransferActionsFeatureContext(s)
TransferChecksFeatureContext(s)
TransferSetupFeatureContext(s)
UsersActionsFeatureContext(s)
UsersSetupFeatureContext(s)
UsersChecksFeatureContext(s)
@ -72,9 +67,7 @@ func ScenarioInitializer(s *godog.ScenarioContext) {
var ctx *testContext.TestContext //nolint[gochecknoglobals]
func beforeScenario(scenarioCtx context.Context, _ *godog.Scenario) (context.Context, error) {
// NOTE(GODT-219) It would be possible to optimised the usage of godog with our context.
app := os.Getenv("TEST_APP")
ctx = testContext.New(app)
ctx = testContext.New()
return scenarioCtx, nil
}

2
test/benchmarks/bench_test.go
View File

@ -27,7 +27,7 @@ import (
)
func benchTestContext() (*context.TestContext, *mocks.IMAPClient) {
ctx := context.New("bridge")
ctx := context.New()
username := "user"
account := ctx.GetTestAccount(username)

35
test/context/context.go
View File

@ -23,8 +23,6 @@ import (
"github.com/ProtonMail/proton-bridge/internal/bridge"
"github.com/ProtonMail/proton-bridge/internal/config/useragent"
"github.com/ProtonMail/proton-bridge/internal/importexport"
"github.com/ProtonMail/proton-bridge/internal/transfer"
"github.com/ProtonMail/proton-bridge/internal/users"
"github.com/ProtonMail/proton-bridge/pkg/listener"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
@ -55,11 +53,10 @@ type TestContext struct {
clientManager pmapi.Manager
// Core related variables.
bridge *bridge.Bridge
importExport *importexport.ImportExport
users *users.Users
credStore users.CredentialsStorer
lastError error
bridge *bridge.Bridge
users *users.Users
credStore users.CredentialsStorer
lastError error
// IMAP related variables.
imapAddr string
@ -75,13 +72,6 @@ type TestContext struct {
smtpLastResponses map[string]*mocks.SMTPResponse
smtpResponseLocker sync.Locker
// Transfer related variables.
transferLocalRootForImport string
transferLocalRootForExport string
transferRemoteIMAPServer *mocks.IMAPServer
transferProgress *transfer.Progress
transferSkipEncryptedMessages bool
// Store releated variables.
bddMessageIDsToAPIIDs map[string]string
@ -93,9 +83,11 @@ type TestContext struct {
}
// New returns a new test TestContext.
func New(app string) *TestContext {
func New() *TestContext {
setLogrusVerbosityFromEnv()
listener := listener.New()
pmapiController, clientManager := newPMAPIController(app, listener)
pmapiController, clientManager := newPMAPIController(listener)
ctx := &TestContext{
t: &bddT{},
@ -121,15 +113,8 @@ func New(app string) *TestContext {
// Ensure that the config is cleaned up after the test is over.
ctx.addCleanupChecked(ctx.locations.Clear, "Cleaning bridge config data")
// Create bridge or import-export instance under test.
switch app {
case "bridge":
ctx.withBridgeInstance()
case "ie":
ctx.withImportExportInstance()
default:
panic("unknown app: " + app)
}
// Create bridge instance under test.
ctx.withBridgeInstance()
return ctx
}

50
test/context/importexport.go
View File

@ -1,50 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package context
import (
"github.com/ProtonMail/proton-bridge/internal/importexport"
"github.com/ProtonMail/proton-bridge/internal/users"
"github.com/ProtonMail/proton-bridge/pkg/listener"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
)
// GetImportExport returns import-export instance.
func (ctx *TestContext) GetImportExport() *importexport.ImportExport {
return ctx.importExport
}
// withImportExportInstance creates a import-export instance for use in the test.
// TestContext has this by default once called with env variable TEST_APP=ie.
func (ctx *TestContext) withImportExportInstance() {
ctx.importExport = newImportExportInstance(ctx.t, ctx.locations, ctx.cache, ctx.credStore, ctx.listener, ctx.clientManager)
ctx.users = ctx.importExport.Users
}
// newImportExportInstance creates a new import-export instance configured to use the given config/credstore.
func newImportExportInstance(
t *bddT,
locations importexport.Locator,
cache importexport.Cacher,
credStore users.CredentialsStorer,
eventListener listener.Listener,
clientManager pmapi.Manager,
) *importexport.ImportExport {
panicHandler := &panicHandler{t: t}
return importexport.New(locations, cache, panicHandler, eventListener, clientManager, credStore)
}

4
test/context/pmapi_controller.go
View File

@ -46,7 +46,7 @@ type PMAPIController interface {
UnlockEvents()
}
func newPMAPIController(app string, listener listener.Listener) (PMAPIController, pmapi.Manager) {
func newPMAPIController(listener listener.Listener) (PMAPIController, pmapi.Manager) {
switch os.Getenv(EnvName) {
case EnvFake:
cntl, cm := fakeapi.NewController()
@ -54,7 +54,7 @@ func newPMAPIController(app string, listener listener.Listener) (PMAPIController
return cntl, cm
case EnvLive:
cntl, cm := liveapi.NewController(app)
cntl, cm := liveapi.NewController()
addConnectionObserver(cm, listener)
return cntl, cm

101
test/context/transfer.go
View File

@ -1,101 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package context
import (
"io/ioutil"
"math/rand"
"os"
"strconv"
"github.com/ProtonMail/proton-bridge/internal/transfer"
"github.com/ProtonMail/proton-bridge/test/mocks"
)
// SetTransferProgress sets transfer progress.
func (ctx *TestContext) SetTransferProgress(progress *transfer.Progress) {
ctx.transferProgress = progress
}
// GetTransferProgress returns transfer progress.
func (ctx *TestContext) GetTransferProgress() *transfer.Progress {
return ctx.transferProgress
}
// SetTransferSkipEncryptedMessages sets whether encrypted messages will be skipped.
func (ctx *TestContext) SetTransferSkipEncryptedMessages(value bool) {
ctx.transferSkipEncryptedMessages = value
}
// GetTransferSkipEncryptedMessages gets whether encrypted messages will be skipped.
func (ctx *TestContext) GetTransferSkipEncryptedMessages() bool {
return ctx.transferSkipEncryptedMessages
}
// GetTransferLocalRootForImport creates temporary root for importing
// if it not exists yet, and returns its path.
func (ctx *TestContext) GetTransferLocalRootForImport() string {
if ctx.transferLocalRootForImport != "" {
return ctx.transferLocalRootForImport
}
root := ctx.createLocalRoot()
ctx.transferLocalRootForImport = root
return root
}
// GetTransferLocalRootForExport creates temporary root for exporting
// if it not exists yet, and returns its path.
func (ctx *TestContext) GetTransferLocalRootForExport() string {
if ctx.transferLocalRootForExport != "" {
return ctx.transferLocalRootForExport
}
root := ctx.createLocalRoot()
ctx.transferLocalRootForExport = root
return root
}
func (ctx *TestContext) createLocalRoot() string {
root, err := ioutil.TempDir("", "transfer")
if err != nil {
panic("failed to create temp transfer root: " + err.Error())
}
ctx.addCleanupChecked(func() error {
return os.RemoveAll(root)
}, "Cleaning transfer data")
return root
}
// GetTransferRemoteIMAPServer creates mocked IMAP server if it not created yet, and returns it.
func (ctx *TestContext) GetTransferRemoteIMAPServer() *mocks.IMAPServer {
if ctx.transferRemoteIMAPServer != nil {
return ctx.transferRemoteIMAPServer
}
port := 21300 + rand.Intn(100) //nolint[gosec] It is OK to use weaker rand generator here
ctx.transferRemoteIMAPServer = mocks.NewIMAPServer("user", "pass", "127.0.0.1", strconv.Itoa(port))
ctx.transferRemoteIMAPServer.Start()
ctx.addCleanupChecked(func() error {
ctx.transferRemoteIMAPServer.Stop()
return nil
}, "Cleaning transfer IMAP server")
return ctx.transferRemoteIMAPServer
}

43
test/features/ie/transfer/export_eml.feature
View File

@ -1,43 +0,0 @@
Feature: Export to EML files
Background:
Given there is connected user "user"
And there is "user" with mailbox "Folders/Foo"
And there are messages in mailbox "INBOX" for "user"
| from | to | subject | time |
| bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 |
And there are messages in mailbox "Folders/Foo" for "user"
| from | to | subject | time |
| foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 |
| bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
| bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |
Scenario: Export all
When user "user" exports to EML files
Then progress result is "OK"
# Every message is also in All Mail.
And transfer exported 8 messages
And transfer imported 8 messages
And transfer failed for 0 messages
And transfer exported messages
| folder | from | to | subject | time |
| Inbox | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 |
| Foo | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 |
| Foo | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
| Foo | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |
| All Mail | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 |
| All Mail | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 |
| All Mail | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
| All Mail | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |
Scenario: Export only Foo with time limit
When user "user" exports to EML files with rules
| source | target | from | to |
| Foo | | 2020-01-01T12:10:00 | 2020-01-01T13:00:00 |
Then progress result is "OK"
And transfer exported 2 messages
And transfer imported 2 messages
And transfer failed for 0 messages
And transfer exported messages
| folder | from | to | subject | time |
| Foo | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
| Foo | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |

43
test/features/ie/transfer/export_mbox.feature
View File

@ -1,43 +0,0 @@
Feature: Export to MBOX files
Background:
Given there is connected user "user"
And there is "user" with mailbox "Folders/Foo"
And there are messages in mailbox "INBOX" for "user"
| from | to | subject | time |
| bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 |
And there are messages in mailbox "Folders/Foo" for "user"
| from | to | subject | time |
| foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 |
| bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
| bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |
Scenario: Export all
When user "user" exports to MBOX files
Then progress result is "OK"
# Every message is also in All Mail.
And transfer exported 8 messages
And transfer imported 8 messages
And transfer failed for 0 messages
And transfer exported messages
| folder | from | to | subject | time |
| Inbox | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 |
| Foo | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 |
| Foo | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
| Foo | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |
| All Mail | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 |
| All Mail | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 |
| All Mail | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
| All Mail | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |
Scenario: Export only Foo with time limit
When user "user" exports to MBOX files with rules
| source | target | from | to |
| Foo | | 2020-01-01T12:10:00 | 2020-01-01T13:00:00 |
Then progress result is "OK"
And transfer exported 2 messages
And transfer imported 2 messages
And transfer failed for 0 messages
And transfer exported messages
| folder | from | to | subject | time |
| Foo | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
| Foo | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |

24
test/features/ie/transfer/import_draft.feature
View File

@ -1,24 +0,0 @@
Feature: Import from EML files
Background:
Given there is connected user "user"
Scenario: Import draft without from fallbacks to primary address
Given there is EML file "Drafts/one.eml"
"""
Subject: no from yet
To: Internal Bridge <test@protonmail.com>
Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
hello
"""
When user "user" imports local files with rules
| source | target |
| Drafts | Drafts |
Then progress result is "OK"
And transfer exported 1 messages
And transfer imported 1 messages
And transfer failed for 0 messages
And API mailbox "Drafts" for "user" has messages
| from | to | subject |
| [userAddress] | test@protonmail.com | no from yet |

43
test/features/ie/transfer/import_embedded.feature
View File

@ -1,43 +0,0 @@
Feature: Import embedded message
Background:
Given there is connected user "user"
And there is EML file "Inbox/hello.eml"
"""
From: Foo <foo@example.com>
To: Bridge Test <bridgetest@pm.test>
Subject: Embedded message
Content-Type: multipart/mixed; boundary="boundary"
Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
This is a multi-part message in MIME format.
--boundary
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 7bit
--boundary
Content-Type: message/rfc822; name="embedded.eml"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="embedded.eml"
From: Bar <bar@example.com>
To: Bridge Test <bridgetest@pm.test>
Subject: (No Subject)
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: quoted-printable
hello
--boundary--
"""
Scenario: Import it
When user "user" imports local files
Then progress result is "OK"
And transfer exported 1 messages
And transfer imported 1 messages
And transfer failed for 0 messages
And API mailbox "INBOX" for "user" has messages
| from | to | subject |
| foo@example.com | bridgetest@pm.test | Embedded message |

61
test/features/ie/transfer/import_eml.feature
View File

@ -1,61 +0,0 @@
Feature: Import from EML files
Background:
Given there is connected user "user"
And there is "user" with mailbox "Folders/Foo"
And there is "user" with mailbox "Folders/Bar"
And there are EML files
| file | from | to | subject | time |
| Foo/one.eml | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 |
| Foo/two.eml | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
| Sub/Foo/three.eml | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |
And there is EML file "Inbox/hello.eml"
"""
Subject: hello
From: Bridge Test <bridgetest@pm.test>
To: Internal Bridge <test@protonmail.com>
Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
hello
"""
Scenario: Import all
When user "user" imports local files
Then progress result is "OK"
And transfer exported 4 messages
And transfer imported 4 messages
And transfer failed for 0 messages
And API mailbox "INBOX" for "user" has messages
| from | to | subject |
| bridgetest@pm.test | test@protonmail.com | hello |
And API mailbox "Folders/Foo" for "user" has messages
| from | to | subject |
| foo@example.com | bridgetest@protonmail.com | one |
| bar@example.com | bridgetest@protonmail.com | two |
| bar@example.com | bridgetest@protonmail.com | three |
Scenario: Import only Foo to Bar with time limit
When user "user" imports local files with rules
| source | target | from | to |
| Foo | Bar | 2020-01-01T12:10:00 | 2020-01-01T13:00:00 |
Then progress result is "OK"
And transfer exported 2 messages
And transfer imported 2 messages
And transfer failed for 0 messages
And API mailbox "Folders/Bar" for "user" has messages
| from | to | subject |
| bar@example.com | bridgetest@protonmail.com | two |
| bar@example.com | bridgetest@protonmail.com | three |
Scenario: Import broken EML message
Given there is EML file "Broken/broken.eml"
"""
Content-type: multipart/mixed
"""
When user "user" imports local files with rules
| source | target |
| Broken | Foo |
Then progress result is "OK"
And transfer exported 1 messages
And transfer imported 0 messages
And transfer failed for 1 messages

188
test/features/ie/transfer/import_encrypted.feature
View File

@ -1,188 +0,0 @@
Feature: Import from EML files
Background:
Given there is connected user "user"
And there is EML file "Inbox/clear.eml"
"""
Subject: clear
From: Bridge Test <bridgetest@pm.test>
To: Internal Bridge <test@protonmail.com>
Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
secret
"""
And there is EML file "Inbox/encrypted.eml"
"""
Subject: encrypted
From: Bridge Test <bridgetest@pm.test>
To: Internal Bridge <test@protonmail.com>
Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
-----BEGIN PGP MESSAGE-----
hQEMA7hGUUsYs0fEAQgA10NwJSNTLm3vpxVtoYBaA9AjFI5H4hKuV3/f2NHbWb2s
k5enK3tEIOYdFdrO+NLrV6weHq3Dgu4er3URTQ62tFwjSJyeXxmY0d9J+JdxibJg
wqZubuSHYzQHpFqJgoUUWEVEsv0Ao8yQo8BE9iybCKoZf6KojROUuE748oxlxJnV
m1XuaVIzgw4xN0GUA5sLLuWeL94b2dZe5SDDQE5POzDgueZ7faefX8U1pGErCRJ0
sO6FSw3SF4NpvrxVESWgCmsG5pcuxE2JqB0UoHnNDcqsW8w1Q+GabAPo6UqHhgIg
56MRCWeou2djHIIj9TMUIVFzSG/HvTYQWVS+i4Z7AdJJAXr53GgbZQznO80Qxwcb
FFdlwOXHuaXqhqCb338jlQWnbcnUsuJWxBAxkHrlP/nluFqPdIBOglC38kdYSBed
3YwuEB9sXV/fcw==
=B05V
-----END PGP MESSAGE-----
"""
And there is EML file "Inbox/encrypted-mime.eml"
"""
Subject: encrypted mime
From: Bridge Test <bridgetest@pm.test>
To: Internal Bridge <test@protonmail.com>
Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
MIME-Version: 1.0
Content-Type: multipart/encrypted; protocol="application/pgp-encrypted"; boundary="WLjzd46aUAiOcuNXjWTJItBZonI56MuAk"
--WLjzd46aUAiOcuNXjWTJItBZonI56MuAk
Content-Type: application/pgp-encrypted
Content-Description: PGP/MIME version identification
--WLjzd46aUAiOcuNXjWTJItBZonI56MuAk
Content-Type: application/octet-stream; name="encrypted.asc"
Content-Description: OpenPGP encrypted message
Content-Disposition: inline; filename="encrypted.asc"
-----BEGIN PGP MESSAGE-----
hQEMA1ppSfinU0f4AQf9HDkojTV3SspsnhaB0HAsKIrUd+AAdSm49ndnJyjYb210
GFIDE/TqcXmoeOcaJIRWaEOZzdcnixplJHjwp5dvDyCaYQSqYxUQ5Z/JfKbtsDyV
HbQzAh833SBCFlNNTnmF/Onu7yRNje1k8U36bY1VUX1QlerT9HDm2QTMRheuPDUR
H9OvGkuBXRpWRSPyXlPONPQOZTbUxvkuMGgDY0N2wt6kKQsrtduQNC157EJOErq0
Zlhu9CnAyezDupMkSoikR1uyxo7GhyXNxi70Ol3tN7E2fnzeBCjUgmliYTABOGSH
nuPpTNk3/YoLEHXK18E/qR3vJTTl6AFIbOcfRCqpQIUBDAOjbxn1yC74AQEH/0kB
CiNDwPepRxwzv3EZT7V0YPuTCD18m9BZ4W5lVEvMNP7HJnCILJT8QJhLQ+AVBUuw
jhJqxAahssOGQ5BVxnWj4qwM+WzBOplH9Zt9bKTie8IdAJsl5GysL19jc4fjnvsK
weBQiR3Y+lEGEBrCajVrUkrXRHyA0fmel8aPfhiHxbh+jRtY8BWdBeX3gIfjwVKf
mMuTmHQ6ERv9CGpIy7mxRF67EIaVRhQzjNNnRlCIqgZHOpS72SKc6DtyCiR+ECjq
UAKNwOjTNZEzAjczyIB5Hkkw1trtVOZEqdacy0CM/SJxjRA8HQ7/0pjtqjOvcpZU
IB6z7IbZLH7krqJY4ZHS6gFvH3B7YOksaQsQL0x4GsdYY4mGUj/18Dzzw2YscUjs
HCOuN9zwAxEIztSQFFZ8vShbpk73fu80X5qRoCQ9708+sKdO92oDY9oZBQkcUl1T
qhpSdApN9mJl2n+uHfSDy63YynhT/bMMrh0AfZjB4ssX9jNkH2knS/FVFUjFUHVh
6boXr0q9xdxt64onx8BrpWOBCqqXjRWUR2n/+y+zw+YgjqUWjpVmsQoF7wQQ3xo6
Yb8y2WguTG9K6m9rS96dOtkXWJgZOVYZ5zlRqdbGZzlfei1890QfnRsNJQhhwkLq
CJV5bhy6AGZxk9JK/RW33g//i2GDfUx4HptRPEgGWu3ZdQskKwyZB7dc6NMtT2as
tOP6z/wgLIPVlLJEY0jXHkmbGf7Oj9JpBSCQBz57rmZunsTgy/jDIuL6mzeuVdYN
lVHqVao23aTZRPaCmwYqWW254oCKaeE7X8nRaQF+9L2nK4YbUf0+KbDGhjnQy8Qg
K1cQt51NcWsM28jNV6Puww7MS+K0NaMjr1fTHdomfHI27C0Dr1e85BWkDnesLqtw
2s5S/8KdYMdBLuzyfT4UQkYTmtxibRXQR9+TxDmNQ/luMuFTCowgGfebAMOCrwU7
NxrgSyuTmAC1Je1glSMMQghHwBCUB2BUCn/vFlMwHdl1waKrUpRaKQRI3iPhMjMw
91Fsv5cUc6uD2pO7vb+vOm2O7+i08KtBpttjk+ANDJjxiGT0V/omlh40T80vN0h6
yk8ZNTq8MqqvLMyH2wKqmmEjll73AWkHATLawRD3ckmlEF+ywc6J91CAYXokWuHc
N7CBL3vRhEJppZ3rmKNw3ani+ThQLTqnGxzxuB+P5IBO6RGXvjYfiUC3Nb0o1Q6X
+QD5BZlvVkklG4bwRdcn87wSlarA8T/nqlZ388ajNaE1Y2+zyJnJyOUEk3nLcgI8
ovaVF/G3PG4yhPR+oOgE7IdWwp+WFa15OF2iLn8ByQa3V8fsWczXHu/iXLyr0KKl
MJCR4bsCv2hcOFTlYSRMyBs+A9gXA9pT+ljv0g7/Z9BuFSmr6pRzgK/guk6WzoTt
m+TxDn1hEovo62KkhAyMtD1hbYO/5GDB6X8tI0YM0kRk8E+H8fuxl43uUE+y9B0X
7Qmkf1Oym9x23S+372MiEa/avAWZTtHhhii37lWkKU+pkx+aiMrfJyozafx6cAaQ
Rxx5uv+8lXEZy4qNEXop7yKDz2agSd6XdZziSIO69BF3x6DMKZdBJtyc5V2RqibU
t80ziVK0IStJmNUPZ1DSMXiwN3yzkQ/bm9RH3x3PPvaVNjISHdl85wlDFc8FM2m0
Q0RM40lj5XAEs1O8iBk5m9yCNMSKQLq5vOhmbygK3ILp4dBoYr6EGZjz+Nq4M+ws
n/dzdR62oCVuKYvVyJVUkmt4DGTo7Pi9ngjAdmLu/RLL8M0/MG9wbu2adT7c2ypj
HM3lUqm+KEf9CdpJBVj0RH5BDWKwDpWx6g6np+GoXsj9nkXYv5qxzVNwgpjTRHwH
xJE+1nFStBtiWunP6eqd8Fl99/jATgVU9ytp+Q+nnZPZn+KHCZEl3CF/TBKsNl7S
QUwdepNF80MDYFi2r685SqM6fvefur0sqyeDwsBOM4GBU88FH9GnWJhQqKVEmQH2
PV/UzkCPpj0ngkQiQjGMQjOKmI6npljOWbIw7LrhggOnfFnP2iTO0B4aAx1h6Ppi
3+jkrdJEuxB89f8P/W8ChtOw7s53YTwYtxmZ+/x0e1G4Nh8pPcFRFF2t/UHEav5v
s3CyH7reAIXDclHH46wbrczvcf6FzS+o8ypIRFAapamUhPqpksuIvyoUeQv8WW/Q
m2tFOPp9wJp/+GAEbuZTyTd/o7Cms3Zl0EOQB9tgqWyqWhasPd40/SCdeXzqpEMS
5Io0tE0ohY9DzN96kn2+07FUSqOYInup3+EXUhCGF8K/i1dny6/o7ZxDjW5xsTdb
AZxd0UEdhvJtvtKhckLhICzImeLGrCUz/zuJBvTR08ir8Rm8kkAmHBn9/jf8+42J
X3TSTes7+k+DtZP6VL6RKhTAzFIEWLQZ+38nzGPfM0BUKf0sGW3wlWFQREU9k9QX
S/idPNOqdNHz/l0eUwf3/bjfAB6OqitPWYH23d6zMkMEgwx/gJmboOfiYu9FKvXJ
tvRgOHb6Rww8fUQhlDOhVupo0DFTIghdeXjeVn/CxIUO67Ns+PI5IB+/sw1KxTIp
kZjjft/l3+mSnyJVyqvzKyfA1WhaXLXJfcJyeGt/Y45RiYnkbSdcbuNhngn+NpZZ
SAcS4vyUqkDQ6RvWU+fww8EYxptNALt9hnc1wD+e8b3Gz2citRrLrc4AhiZwafp8
gj4HtKBXFz3oX2vhHgubTLuiEKhGc2dYXL9Z2PlXOWZhauTO00iVYfbWPsdTRSvi
QmKIP9QVzDq6StfHxs2x47yxrHrEtCjsHjDz3d+r+p6i6O212EHlCQewaPfAieBp
lw01cJB1KwyeoYgTczQkz6hhM+fj1RBMNDqxTHBVb3GGNh1nxu+4UR+tgQG/V/Ot
M/1NE8+yeRktzukDX1toXfCFXvRL3ijriHliaivWww==
=4M3l
-----END PGP MESSAGE-----
--WLjzd46aUAiOcuNXjWTJItBZonI56MuAk--
"""
And there is EML file "Inbox/signed.eml"
"""
Subject: signed
From: Bridge Test <bridgetest@pm.test>
To: Internal Bridge <test@protonmail.com>
Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256
secret
-----BEGIN PGP SIGNATURE-----
iQEzBAEBCAAdFiEENaE2ZPemenI4pZah/SJcGo7SJWIFAl+cCAgACgkQ/SJcGo7S
JWKsOQf/YakNXkMNjZIu8Hf1WflxtiDXVzTugOicC05k5W64oIqSHt0xNaFKE37k
//3eDMWbHvqHKFVdg7qcLsVPeVBaW3bdZUiexGM24OiGgyEitufnHQLOtEDTound
JyH5nUeHpvpBKIIOJZNBDM0HsRYnwKwrOWk3N2VRwog4J8J3cmJ/f9bPWNI/0OPT
qmtVGRVg6Ge83nZn51Vof//jFzkO4wGYCsE0aF0Ywc7nISZuyKQzmu/qgmwzDG50
PjpvIQ/ygisRPNdRlylXEqyoIDCQ+v0AnxhhwX/5dbt6xMuMMOxBrFSC94Zce1Vj
x2ssXlT4ONPnkI/YWwhtQPLU628IMg==
=GiS3
-----END PGP SIGNATURE-----
"""
And there is EML file "Inbox/encrypted-signed.eml"
"""
Subject: encrypted and signed
From: Bridge Test <bridgetest@pm.test>
To: Internal Bridge <test@protonmail.com>
Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
-----BEGIN PGP MESSAGE-----
hQEMA7hGUUsYs0fEAQf/dppHciWIf+o4l0gEfHeyHV/HVhG4es0aVQYrwFQlSWVx
estMuyLBSMfrsQXLago7Q9ZNo/XnKszzprCXxxYH52hAg64oAsjKB3jgRmVizs8b
8lj0BRf003wUluS/0msV9SiEZBGeL8jGq6Te9vaM8OHHhIVzVjGnRdTSC0jBE6cS
vy8IBHXYe0LfdZiPojPDPGQdSej+H3uu7eZGBvVHTDeQLPDel4k7Ykdr0qlNXs6O
5XpM5YG4w+t0aG+YROPH+BUj8PpPojQ/lrv/yFISTRbHlEd8N50w8BNTnBet+9Vm
oPcyvN+RQxBlvRuPpDjUmREvmtObKZV6+m6gocemx9LAzQEeVLcpjO/hJhl8gX72
MNz3McU7aXf5sSoOPdHDNx8T2NON/2bwG5FE+PRMuVywTKhCB7o8VAsJpGMQ8xRM
5WCNhow0AI7kni8yZA+GbvspnJWfit9tCTR5MIFHCSH9J3kJJnWkxQSN04GGpBcd
n43GWn7O7ufA4lMMZiGXMdi/J1iV9waAsIfMPk29BMq6xK0/jJYdHqQS+vNsSnF5
xL/Ir4RYq4SFFA06A/E7HpXr2ruZhBQCkzaIIdrVJR/Lp2VLJIVulTBQK8y2AFtj
JeeKS0kIuC/7UPF2O624kwNr8dmIhDJYusFs6ZeED/nAKwDO/vP2CSwVC3sUjn3N
u+sWqQUTxSmjhRVf9b0+VyTh0mXCovJQXomL6Zz6lxXuJqqzELIOfCxYD1z9GwTG
cT08Aa2eEpf3agdLCTxvjO3iq9FksMHvIN+LSCQ6Pw+aTByjrk0oMmvGbANAogTk
yrplG/iRVlmq0p/Cfl5UEjKqT/nt5j9zbpeuYXmhjiBT9SBE07oUVLY1VT7ihcY=
=qYnL
-----END PGP MESSAGE-----
"""
Scenario: Import encrypted
Given there is skip encrypted messages set to "false"
When user "user" imports local files
Then progress result is "OK"
And transfer failed for 0 messages
And transfer exported 5 messages
And transfer imported 5 messages
And transfer skipped 0 messages
And API mailbox "INBOX" for "user" has messages
| from | to | subject |
| bridgetest@pm.test | test@protonmail.com | clear |
| bridgetest@pm.test | test@protonmail.com | encrypted |
| bridgetest@pm.test | test@protonmail.com | encrypted mime |
| bridgetest@pm.test | test@protonmail.com | signed |
| bridgetest@pm.test | test@protonmail.com | encrypted and signed |
Scenario: Skip encrypted
Given there is skip encrypted messages set to "true"
When user "user" imports local files
Then progress result is "OK"
And transfer failed for 0 messages
And transfer exported 5 messages
And transfer imported 2 messages
And transfer skipped 3 messages
And API mailbox "INBOX" for "user" has messages
| from | to | subject |
| bridgetest@pm.test | test@protonmail.com | clear |
| bridgetest@pm.test | test@protonmail.com | signed |

49
test/features/ie/transfer/import_export.feature
View File

@ -1,49 +0,0 @@
Feature: Import-Export app
Background:
Given there is connected user "user"
And there is "user" with mailbox "Folders/Foo"
And there is "user" with mailbox "Folders/Bar"
Scenario: EML -> PM -> EML
Given there are EML files
| file | from | to | subject | time |
| Inbox/hello.eml | bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 |
| Foo/one.eml | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 |
| Foo/two.eml | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
| Sub/Foo/three.eml | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |
When user "user" imports local files
Then progress result is "OK"
And transfer failed for 0 messages
And transfer imported 4 messages
When user "user" exports to EML files
Then progress result is "OK"
And transfer failed for 0 messages
# Every message is also in All Mail.
And transfer imported 8 messages
And exported messages match the original ones
Scenario: MBOX -> PM -> MBOX
Given there is MBOX file "Inbox.mbox" with messages
| from | to | subject | time |
| bridgetest@pm.test | test@protonmail.com | hello | 2020-01-01T12:00:00 |
And there is MBOX file "Foo.mbox" with messages
| from | to | subject | time |
| foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 |
| bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
| bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |
When user "user" imports local files
Then progress result is "OK"
And transfer failed for 0 messages
And transfer imported 4 messages
When user "user" exports to MBOX files
Then progress result is "OK"
And transfer failed for 0 messages
# Every message is also in All Mail.
And transfer imported 8 messages
And exported messages match the original ones

80
test/features/ie/transfer/import_imap.feature
View File

@ -1,80 +0,0 @@
Feature: Import from IMAP server
Background:
Given there is connected user "user"
And there is "user" with mailbox "Folders/Foo"
And there is "user" with mailbox "Folders/Bar"
And there are IMAP mailboxes
| name |
| Inbox |
| Foo |
| Broken |
And there are IMAP messages
| mailbox | seqnum | uid | from | to | subject | time |
| Foo | 1 | 12 | foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 |
| Foo | 2 | 14 | bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
| Foo | 3 | 15 | bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |
And there is IMAP message in mailbox "Inbox" with seq 1, uid 42, time "2020-01-01T12:34:56" and subject "hello"
"""
Subject: hello
From: Bridge Test <bridgetest@pm.test>
To: Internal Bridge <test@protonmail.com>
Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
hello
"""
Scenario: Import all
When user "user" imports remote messages
Then progress result is "OK"
And transfer exported 4 messages
And transfer imported 4 messages
And transfer failed for 0 messages
And API mailbox "INBOX" for "user" has messages
| from | to | subject |
| bridgetest@pm.test | test@protonmail.com | hello |
And API mailbox "Folders/Foo" for "user" has messages
| from | to | subject |
| foo@example.com | bridgetest@protonmail.com | one |
| bar@example.com | bridgetest@protonmail.com | two |
| bar@example.com | bridgetest@protonmail.com | three |
Scenario: Import only Foo to Bar with time limit
When user "user" imports remote messages with rules
| source | target | from | to |
| Foo | Bar | 2020-01-01T12:10:00 | 2020-01-01T13:00:00 |
Then progress result is "OK"
And transfer exported 2 messages
And transfer imported 2 messages
And transfer failed for 0 messages
And API mailbox "Folders/Bar" for "user" has messages
| from | to | subject |
| bar@example.com | bridgetest@protonmail.com | two |
| bar@example.com | bridgetest@protonmail.com | three |
# Note we need to have message which we can parse and use in go-imap
# but which has problem on our side. Used example with missing boundary
# is real example which we want to solve one day. Probabl this test
# can be removed once we import any time of message or switch is to
# something we will never allow.
Scenario: Import broken message
Given there is IMAP message in mailbox "Broken" with seq 1, uid 42, time "2020-01-01T12:34:56" and subject "broken"
"""
Subject: missing boundary end
Content-Type: multipart/related; boundary=boundary
--boundary
Content-Disposition: inline
Content-Transfer-Encoding: quoted-printable
Content-Type: text/plain; charset=utf-8
body
"""
When user "user" imports remote messages with rules
| source | target |
| Broken | Foo |
Then progress result is "OK"
And transfer exported 1 messages
And transfer imported 0 messages
And transfer failed for 1 messages

65
test/features/ie/transfer/import_mbox.feature
View File

@ -1,65 +0,0 @@
Feature: Import from MBOX files
Background:
Given there is connected user "user"
And there is "user" with mailbox "Folders/Foo"
And there is "user" with mailbox "Folders/Bar"
And there is MBOX file "Foo.mbox" with messages
| from | to | subject | time |
| foo@example.com | bridgetest@protonmail.com | one | 2020-01-01T12:00:00 |
| bar@example.com | bridgetest@protonmail.com | two | 2020-01-01T13:00:00 |
And there is MBOX file "Sub/Foo.mbox" with messages
| from | to | subject | time |
| bar@example.com | bridgetest@protonmail.com | three | 2020-01-01T12:30:00 |
And there is MBOX file "Inbox.mbox"
"""
From bridgetest@pm.test Thu Feb 20 20:20:20 2020
Subject: hello
From: Bridge Test <bridgetest@pm.test>
To: Internal Bridge <test@protonmail.com>
Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
hello
"""
Scenario: Import all
When user "user" imports local files
Then progress result is "OK"
And transfer exported 4 messages
And transfer imported 4 messages
And transfer failed for 0 messages
And API mailbox "INBOX" for "user" has messages
| from | to | subject |
| bridgetest@pm.test | test@protonmail.com | hello |
And API mailbox "Folders/Foo" for "user" has messages
| from | to | subject |
| foo@example.com | bridgetest@protonmail.com | one |
| bar@example.com | bridgetest@protonmail.com | two |
| bar@example.com | bridgetest@protonmail.com | three |
Scenario: Import only Foo to Bar with time limit
When user "user" imports local files with rules
| source | target | from | to |
| Foo | Bar | 2020-01-01T12:10:00 | 2020-01-01T13:00:00 |
Then progress result is "OK"
And transfer exported 2 messages
And transfer imported 2 messages
And transfer failed for 0 messages
And API mailbox "Folders/Bar" for "user" has messages
| from | to | subject |
| bar@example.com | bridgetest@protonmail.com | two |
| bar@example.com | bridgetest@protonmail.com | three |
Scenario: Import broken message
Given there is MBOX file "Broken.mbox"
"""
From bridgetest@pm.test Thu Feb 20 20:20:20 2020
Content-type: multipart/mixed
"""
When user "user" imports local files with rules
| source | target |
| Broken | Foo |
Then progress result is "OK"
And transfer exported 1 messages
And transfer imported 0 messages
And transfer failed for 1 messages

98
test/features/ie/transfer/import_sent.feature
View File

@ -1,98 +0,0 @@
Feature: Import to sent
Background:
Given there is connected user "user"
And there is "user" with mailbox "Labels/label"
And there is EML file "Sent/one.eml"
"""
Subject: one
From: Foo <foo@example.com>
To: Bridge Test <bridgetest@pm.test>
Message-ID: one.integrationtest
one
"""
And there is EML file "Sent/two.eml"
"""
Subject: two
From: Bar <bar@example.com>
To: Bridge Test <bridgetest@pm.test>
Message-ID: two.integrationtest
two
"""
Scenario: Import sent only
When user "user" imports local files
Then progress result is "OK"
And transfer exported 2 messages
And transfer imported 2 messages
And transfer failed for 0 messages
And API mailbox "INBOX" for "user" has 0 message
And API mailbox "Sent" for "user" has messages
| from | to | subject |
| foo@example.com | bridgetest@pm.test | one |
| bar@example.com | bridgetest@pm.test | two |
# Messages imported to label only are added automatically to Archive folder.
# Then it depends on the order: if the message is first imported to Sent
# folder and later to that label with importing to Archive, message will not
# be in Sent but Archive. The order is semi-random for the big messages,
# e.g., it will do alphabetical order of mailboxes, but for under ten small
# messages the order is random every time (because we are importing in
# batches of up to ten messages and iterating through map we use to collect
# messages is random). So we cannot for this test ensure the same output
# every time.
@ignore-live
Scenario: Import to sent and custom label
And there is EML file "Label/one.eml"
"""
Subject: one
From: Foo <foo@example.com>
To: Bridge Test <bridgetest@pm.test>
Message-ID: one.integrationtest
one
"""
When user "user" imports local files
Then progress result is "OK"
And transfer exported 3 messages
And transfer imported 3 messages
And transfer failed for 0 messages
# We had an issue that moving message to Sent automatically added
# the message also into Inbox if the message was in some custom label.
And API mailbox "INBOX" for "user" has 0 message
And API mailbox "Labels/label" for "user" has messages
| from | to | subject |
| foo@example.com | bridgetest@pm.test | one |
And API mailbox "Sent" for "user" has messages
| from | to | subject |
| foo@example.com | bridgetest@pm.test | one |
| bar@example.com | bridgetest@pm.test | two |
Scenario: Import to sent and inbox is in both mailboxes
And there is EML file "Inbox/one.eml"
"""
Subject: one
From: Foo <foo@example.com>
To: Bridge Test <bridgetest@pm.test>
Message-ID: one.integrationtest
Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
one
"""
When user "user" imports local files
Then progress result is "OK"
And transfer exported 3 messages
And transfer imported 3 messages
And transfer failed for 0 messages
And API mailbox "INBOX" for "user" has messages
| from | to | subject |
| foo@example.com | bridgetest@pm.test | one |
And API mailbox "Sent" for "user" has messages
| from | to | subject |
| foo@example.com | bridgetest@pm.test | one |
| bar@example.com | bridgetest@pm.test | two |

20
test/features/ie/users/delete.feature
View File

@ -1,20 +0,0 @@
Feature: Delete user
Scenario: Deleting connected user
Given there is connected user "user"
When user deletes "user"
Then last response is "OK"
Scenario: Deleting connected user with cache
Given there is connected user "user"
When user deletes "user" with cache
Then last response is "OK"
Scenario: Deleting disconnected user
Given there is disconnected user "user"
When user deletes "user"
Then last response is "OK"
Scenario: Deleting disconnected user with cache
Given there is disconnected user "user"
When user deletes "user" with cache
Then last response is "OK"

58
test/features/ie/users/login.feature
View File

@ -1,58 +0,0 @@
Feature: Login for the first time
Scenario: Normal login
Given there is user "user"
When "user" logs in
Then last response is "OK"
And "user" is connected
@ignore-live
Scenario: Login with bad username
When "user" logs in with bad password
Then last response is "failed to login: Incorrect login credentials. Please try again"
@ignore-live
Scenario: Login with bad password
Given there is user "user"
When "user" logs in with bad password
Then last response is "failed to login: Incorrect login credentials. Please try again"
Scenario: Login without internet connection
Given there is no internet connection
When "user" logs in
Then last response is "failed to login: no internet connection"
@ignore-live
Scenario: Login user with 2FA
Given there is user "user2fa"
When "user2fa" logs in
Then last response is "OK"
And "user2fa" is connected
Scenario: Login user with capital letters in address
Given there is user "userAddressWithCapitalLetter"
When "userAddressWithCapitalLetter" logs in
Then last response is "OK"
And "userAddressWithCapitalLetter" is connected
Scenario: Login user with more addresses
Given there is user "userMoreAddresses"
When "userMoreAddresses" logs in
Then last response is "OK"
And "userMoreAddresses" is connected
@ignore-live
Scenario: Login user with disabled primary address
Given there is user "userDisabledPrimaryAddress"
When "userDisabledPrimaryAddress" logs in
Then last response is "OK"
And "userDisabledPrimaryAddress" is connected
Scenario: Login two users
Given there is user "user"
And there is user "userMoreAddresses"
When "user" logs in
Then last response is "OK"
And "user" is connected
When "userMoreAddresses" logs in
Then last response is "OK"
And "userMoreAddresses" is connected

12
test/features/ie/users/relogin.feature
View File

@ -1,12 +0,0 @@
Feature: Re-login
Scenario: Re-login with connected user
Given there is connected user "user"
When "user" logs in
Then last response is "failed to finish login: user is already connected"
And "user" is connected
Scenario: Re-login with disconnected user
Given there is disconnected user "user"
When "user" logs in
Then last response is "OK"
And "user" is connected

0
test/features/bridge/imap/auth.feature → test/features/imap/auth.feature
View File

0
test/features/bridge/imap/idle/basic.feature → test/features/imap/idle/basic.feature
View File

0
test/features/bridge/imap/idle/two_users.feature → test/features/imap/idle/two_users.feature
View File

0
test/features/bridge/imap/mailbox/create.feature → test/features/imap/mailbox/create.feature
View File

0
test/features/bridge/imap/mailbox/delete.feature → test/features/imap/mailbox/delete.feature
View File

0
test/features/bridge/imap/mailbox/info.feature → test/features/imap/mailbox/info.feature
View File

0
test/features/bridge/imap/mailbox/list.feature → test/features/imap/mailbox/list.feature
View File

0
test/features/bridge/imap/mailbox/rename.feature → test/features/imap/mailbox/rename.feature
View File

0
test/features/bridge/imap/mailbox/select.feature → test/features/imap/mailbox/select.feature
View File

Some files were not shown because too many files have changed in this diff Show More