From 658ead9fb39b5074e8944f642ce06d5a399d3e6a Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 12 Aug 2020 13:56:49 +0200 Subject: [PATCH] Import/Export final touches --- .gitlab-ci.yml | 62 ++++--------- Changelog.md | 56 +++--------- Makefile | 16 ++-- README.md | 12 ++- cmd/Desktop-Bridge/main.go | 10 +-- cmd/Import-Export/main.go | 33 +++---- doc/importexport.md | 20 ++--- doc/index.md | 4 +- internal/bridge/credits.go | 4 +- .../cmd/{memory_profile.go => profiles.go} | 15 +++- internal/cmd/version_file.go | 2 +- internal/frontend/cli-ie/account_utils.go | 2 +- internal/frontend/cli-ie/accounts.go | 2 +- internal/frontend/cli-ie/frontend.go | 18 ++-- internal/frontend/cli-ie/importexport.go | 2 +- internal/frontend/cli-ie/system.go | 14 ++- internal/frontend/cli-ie/updates.go | 6 +- internal/frontend/cli-ie/utils.go | 6 +- internal/frontend/cli/frontend.go | 2 +- internal/frontend/cli/system.go | 2 +- internal/frontend/cli/updates.go | 2 +- .../qml/ImportExportUI/DialogImport.qml | 2 +- .../qml/ImportExportUI/ImportDelegate.qml | 20 +++-- .../qml/ImportExportUI/ImportStructure.qml | 2 +- .../qml/ImportExportUI/MainWindow.qml | 10 ++- .../qml/ImportExportUI/SelectFolderMenu.qml | 1 - .../frontend/qml/ProtonUI/PopupMessage.qml | 3 +- internal/frontend/qml/tst_GuiIE.qml | 18 ++-- internal/frontend/qt-common/common.go | 10 --- internal/frontend/qt-ie/export.go | 63 +++---------- internal/frontend/qt-ie/folder_functions.go | 18 ++-- internal/frontend/qt-ie/frontend.go | 17 ++-- internal/frontend/qt-ie/import.go | 2 + internal/frontend/qt-ie/mbox.go | 33 ++++--- .../{qt-common => qt-ie}/path_status.go | 4 +- internal/frontend/qt-ie/transfer_rules.go | 51 ++++++++--- internal/frontend/qt-ie/ui.go | 3 +- internal/frontend/qt/frontend.go | 6 +- internal/frontend/types/types.go | 8 +- internal/importexport/importexport.go | 12 +-- internal/importexport/store_factory.go | 2 +- internal/importexport/types.go | 1 + internal/metrics/metrics.go | 2 +- internal/transfer/mailbox.go | 65 +++++++++----- internal/transfer/mailbox_test.go | 7 ++ internal/transfer/progress.go | 85 +++++++++++------- internal/transfer/progress_test.go | 8 +- internal/transfer/provider_eml_source.go | 5 ++ internal/transfer/provider_eml_target.go | 3 +- internal/transfer/provider_imap.go | 6 +- internal/transfer/provider_imap_source.go | 13 ++- internal/transfer/provider_mbox_source.go | 5 ++ internal/transfer/provider_pmapi_source.go | 38 ++++---- internal/transfer/provider_pmapi_target.go | 5 ++ internal/transfer/rules.go | 4 +- internal/transfer/rules_test.go | 2 +- internal/transfer/transfer.go | 20 +++-- {pkg => internal}/updates/bridge_pubkey.gpg | 0 {pkg => internal}/updates/compare_versions.go | 0 .../updates/compare_versions_test.go | 0 {pkg => internal}/updates/downloader.go | 0 {pkg => internal}/updates/progress.go | 0 {pkg => internal}/updates/signature.go | 0 {pkg => internal}/updates/sync.go | 0 {pkg => internal}/updates/sync_test.go | 0 {pkg => internal}/updates/tar.go | 0 .../testdata/current_version_linux.json | 0 .../testdata/current_version_linux.json.sig | Bin {pkg => internal}/updates/updates.go | 4 +- {pkg => internal}/updates/updates_beta.go | 0 {pkg => internal}/updates/updates_qa.go | 0 {pkg => internal}/updates/updates_test.go | 0 {pkg => internal}/updates/version_info.go | 0 internal/users/credentials/credentials.go | 2 +- internal/users/users.go | 11 ++- pkg/config/config.go | 6 +- pkg/message/build.go | 10 +-- pkg/pmapi/addresses.go | 9 ++ pkg/pmapi/clientmanager.go | 5 +- test/context/context.go | 2 +- test/context/importexport.go | 6 +- .../ie/transfer/import_export.feature | 2 +- 82 files changed, 451 insertions(+), 450 deletions(-) rename internal/cmd/{memory_profile.go => profiles.go} (75%) rename internal/frontend/{qt-common => qt-ie}/path_status.go (98%) rename {pkg => internal}/updates/bridge_pubkey.gpg (100%) rename {pkg => internal}/updates/compare_versions.go (100%) rename {pkg => internal}/updates/compare_versions_test.go (100%) rename {pkg => internal}/updates/downloader.go (100%) rename {pkg => internal}/updates/progress.go (100%) rename {pkg => internal}/updates/signature.go (100%) rename {pkg => internal}/updates/sync.go (100%) rename {pkg => internal}/updates/sync_test.go (100%) rename {pkg => internal}/updates/tar.go (100%) rename {pkg => internal}/updates/testdata/current_version_linux.json (100%) rename {pkg => internal}/updates/testdata/current_version_linux.json.sig (100%) rename {pkg => internal}/updates/updates.go (98%) rename {pkg => internal}/updates/updates_beta.go (100%) rename {pkg => internal}/updates/updates_qa.go (100%) rename {pkg => internal}/updates/updates_test.go (100%) rename {pkg => internal}/updates/version_info.go (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bd5c1c5d..5a52b7b0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,34 +75,33 @@ dependency-updates: # Stage: BUILD -build-linux: +.build-base: stage: build only: - branches script: - make build + artifacts: + expire_in: 2 week + +build-linux: + extends: .build-base artifacts: name: "bridge-linux-$CI_COMMIT_SHORT_SHA" paths: - bridge_*.tgz - expire_in: 2 week build-ie-linux: - stage: build - only: - - branches + extends: .build-base script: - make build-ie artifacts: name: "ie-linux-$CI_COMMIT_SHORT_SHA" paths: - ie_*.tgz - expire_in: 2 week -build-darwin: - stage: build - only: - - branches +.build-darwin-base: + extends: .build-base before_script: - eval $(ssh-agent -s) - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null @@ -117,47 +116,32 @@ build-darwin: cache: {} tags: - macOS - script: - - make build + +build-darwin: + extends: .build-darwin-base artifacts: name: "bridge-darwin-$CI_COMMIT_SHORT_SHA" paths: - bridge_*.tgz - expire_in: 2 week build-ie-darwin: - stage: build - only: - - branches - before_script: - - eval $(ssh-agent -s) - - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null - - export PATH=/usr/local/bin:$PATH - - export PATH=/usr/local/opt/git/bin:$PATH - - export PATH=/usr/local/opt/make/libexec/gnubin:$PATH - - export PATH=/usr/local/opt/go@1.13/bin:$PATH - - export PATH=/usr/local/opt/gnu-sed/libexec/gnubin:$PATH - - export GOPATH=~/go - - export PATH=$GOPATH/bin:$PATH - cache: {} - tags: - - macOS-bridge + extends: .build-darwin-base script: - make build-ie artifacts: name: "ie-darwin-$CI_COMMIT_SHORT_SHA" paths: - ie_*.tgz - expire_in: 2 week -build-windows: - stage: build +.build-windows-base: + extends: .build-base services: - docker:dind - only: - - branches variables: DOCKER_HOST: tcp://docker:2375 + +build-windows: + extends: .build-windows-base script: # We need to install docker because qtdeploy builds for windows inside a docker container. # Docker will connect to the dockerd daemon provided by the runner service docker:dind at tcp://docker:2375. @@ -170,16 +154,9 @@ build-windows: name: "bridge-windows-$CI_COMMIT_SHORT_SHA" paths: - bridge_*.tgz - expire_in: 2 week build-ie-windows: - stage: build - services: - - docker:dind - only: - - branches - variables: - DOCKER_HOST: tcp://docker:2375 + extends: .build-windows-base script: # We need to install docker because qtdeploy builds for windows inside a docker container. # Docker will connect to the dockerd daemon provided by the runner service docker:dind at tcp://docker:2375. @@ -192,7 +169,6 @@ build-ie-windows: name: "ie-windows-$CI_COMMIT_SHORT_SHA" paths: - ie_*.tgz - expire_in: 2 week # Stage: MIRROR diff --git a/Changelog.md b/Changelog.md index 7e6bf64b..5fd820b2 100644 --- a/Changelog.md +++ b/Changelog.md @@ -35,6 +35,18 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/) * Set first-start to false in bridge, not in frontend. * GODT-400 Refactor sendingInfo. +* GODT-380 Adding IE GUI to Bridge repo and building + * BR: extend functionality of PopupDialog + * BR: makefile APP_VERSION instead of BRIDGE_VERSION + * BR: use common logs function for Qt + * BR: change `go.progressDescription` to `string` + * IE: Rounded button has fa-icon + * IE: `Upgrade` → `Update` + * IE: Moving `AccountModel` to `qt-common` + * IE: Added `ReportBug` to `internal/importexport` + * IE: Added event watch in GUI + * IE: Removed `onLoginFinished` + * Structure for transfer rules in QML ### Fixed * GODT-454 Fix send on closed channel when receiving unencrypted send confirmation from GUI. @@ -47,8 +59,6 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/) * GODT-554 Detect and notify about "bad certificate" IMAP TLS error. * IMAP mailbox info update when new mailbox is created. * GODT-72 Use ISO-8859-1 encoding if charset is not specified and it isn't UTF-8. -* Structure for transfer rules in QML -* GODT-360 Detect charset embedded in html/xml. ### Changed * GODT-360 Detect charset embedded in html/xml. @@ -62,48 +72,6 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/) * GODT-280 Migrate to gopenpgp v2. * `Unlock()` call on pmapi-client unlocks both User keys and Address keys. * Salt is available via `AuthSalt()` method. -* GODT-394 Don't check SMTP message send time in integration tests. -* GODT-380 Adding IE GUI to Bridge repo -* GODT-380 Adding IE GUI to Bridge repo and building - * BR: extend functionality of PopupDialog - * BR: makefile APP_VERSION instead of BRIDGE_VERSION - * BR: use common logs function for Qt - * BR: change `go.progressDescription` to `string` - * IE: Rounded button has fa-icon - * IE: `Upgrade` → `Update` - * IE: Moving `AccountModel` to `qt-common` - * IE: Added `ReportBug` to `internal/importexport` - * IE: Added event watch in GUI - * IE: Removed `onLoginFinished` -* GODT-388 support for both bridge and import/export credentials by package users -* GODT-387 store factory to make store optional -* GODT-386 renamed bridge to general users and keep bridge only for bridge stuff -* GODT-308 better user error message when request is canceled -* GODT-312 validate recipient emails in send before asking for their public keys - -### Fixed -* GODT-356 Fix crash when removing account while mail client is fetching messages (regression from GODT-204). -* GODT-390 Don't logout user if AuthRefresh fails because internet was off. -* GODT-358 Bad timeouts with Alternative Routing. -* GODT-363 Drafts are not deleted when already created on webapp. -* GODT-390 Don't logout user if AuthRefresh fails because internet was off. -* GODT-341 Fixed flaky unittest for Store synchronization cooldown. -* Crash when failing to match necessary html element. -* Crash in message.combineParts when copying nil slice. -* Handle double charset better by using local ParseMediaType instead of mime.ParseMediaType. -* Don't remove log dir. -* GODT-422 Fix element not found (avoid listing credentials, prefer getting). -* GODT-404 Don't keep connections to proxy servers alive if user disables DoH. -* Ensure DoH is used at startup to load users for the initial auth. -* Issue causing deadlock when reloading users keys due to double-locking of a mutex. - -## [v1.2.7] Donghai-hotfix - beta (2020-05-07) - -### Added -* IMAP mailbox info update when new mailbox is created. -* GODT-72 Use ISO-8859-1 encoding if charset is not specified and it isn't UTF-8. - -### Changed * GODT-308 Better user error message when request is canceled. * GODT-162 User Agent does not contain bridge version, only client in format `client name/client version (os)`. * GODT-258 Update go-imap to v1. diff --git a/Makefile b/Makefile index 56f40189..4407c70c 100644 --- a/Makefile +++ b/Makefile @@ -176,12 +176,10 @@ test: gofiles ./internal/smtp/... \ ./internal/store/... \ ./internal/transfer/... \ + ./internal/updates/... \ ./internal/users/... \ ./pkg/... -test-ie: - go test ./internal/transfer/... - bench: go test -run '^$$' -bench=. -memprofile bench_mem.pprof -cpuprofile bench_cpu.pprof ./internal/store go tool pprof -png -output bench_mem.png bench_mem.pprof @@ -228,7 +226,8 @@ gofiles: ./internal/bridge/credits.go ./internal/bridge/release_notes.go ./inter ## Run and debug -.PHONY: run run-ie run-qt run-ie-qt run-qt-cli run-nogui run-ie-nogui run-nogui-cli run-debug run-qml-preview run-ie-qml-preview clean-fronted-qt clean-fronted-qt-ie clean-fronted-qt-common clean +.PHONY: run run-qt run-qt-cli run-nogui run-nogui-cli run-debug run-qml-preview run-ie-qml-preview run-ie run-ie-qt run-ie-qt-cli run-ie-nogui run-ie-nogui-cli clean-vendor clean-frontend-qt clean-frontend-qt-ie clean-frontend-qt-common clean + VERBOSITY?=debug-client RUN_FLAGS:=-m -l=${VERBOSITY} @@ -249,27 +248,24 @@ run-debug: run-qml-preview: $(MAKE) -C internal/frontend/qt -f Makefile.local qmlpreview - run-ie-qml-preview: $(MAKE) -C internal/frontend/qt-ie -f Makefile.local qmlpreview run-ie: TARGET_CMD=Import-Export $(MAKE) run -run-ie-nogui: - TARGET_CMD=Import-Export $(MAKE) run-nogui run-ie-qt: TARGET_CMD=Import-Export $(MAKE) run-qt +run-ie-nogui: + TARGET_CMD=Import-Export $(MAKE) run-nogui + clean-frontend-qt: $(MAKE) -C internal/frontend/qt -f Makefile.local clean - clean-frontend-qt-ie: $(MAKE) -C internal/frontend/qt-ie -f Makefile.local clean - clean-frontend-qt-common: $(MAKE) -C internal/frontend/qt-common -f Makefile.local clean - clean-vendor: clean-frontend-qt clean-frontend-qt-ie clean-frontend-qt-common rm -rf ./vendor diff --git a/README.md b/README.md index f512e1d6..b9d90831 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,16 @@ background. More details [on the public website](https://protonmail.com/bridge). -## Description Import-Export -TODO +## Description Import-Export app +ProtonMail Import-Export app for importing and exporting messages. + +To transfer messages, firstly log in using your ProtonMail credentials. +For import, expand your account, and pick the address to which to import +messages from IMAP server or local EML or MBOX files. For export, pick +the whole account or only a specific address. Then, in both cases, +configure transfer rules (match source and target mailboxes, set time +range limits and so on) and hit start. Once the transfer is complete, +check the results. ## Keychain You need to have a keychain in order to run the ProtonMail Bridge. On Mac or diff --git a/cmd/Desktop-Bridge/main.go b/cmd/Desktop-Bridge/main.go index 0dc459b3..2377ebe9 100644 --- a/cmd/Desktop-Bridge/main.go +++ b/cmd/Desktop-Bridge/main.go @@ -48,12 +48,12 @@ import ( "github.com/ProtonMail/proton-bridge/internal/imap" "github.com/ProtonMail/proton-bridge/internal/preferences" "github.com/ProtonMail/proton-bridge/internal/smtp" + "github.com/ProtonMail/proton-bridge/internal/updates" "github.com/ProtonMail/proton-bridge/internal/users/credentials" "github.com/ProtonMail/proton-bridge/pkg/config" "github.com/ProtonMail/proton-bridge/pkg/constants" "github.com/ProtonMail/proton-bridge/pkg/listener" "github.com/ProtonMail/proton-bridge/pkg/pmapi" - "github.com/ProtonMail/proton-bridge/pkg/updates" "github.com/allan-simon/go-singleinstance" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -168,13 +168,7 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen] // In case user wants to do CPU or memory profiles... if doCPUProfile := context.GlobalBool("cpu-prof"); doCPUProfile { - f, err := os.Create("cpu.pprof") - if err != nil { - log.Fatal("Could not create CPU profile: ", err) - } - if err := pprof.StartCPUProfile(f); err != nil { - log.Fatal("Could not start CPU profile: ", err) - } + cmd.StartCPUProfile() defer pprof.StopCPUProfile() } diff --git a/cmd/Import-Export/main.go b/cmd/Import-Export/main.go index ac653ca3..a2d02cbb 100644 --- a/cmd/Import-Export/main.go +++ b/cmd/Import-Export/main.go @@ -18,19 +18,18 @@ package main import ( - "os" "runtime/pprof" "github.com/ProtonMail/proton-bridge/internal/cmd" "github.com/ProtonMail/proton-bridge/internal/events" "github.com/ProtonMail/proton-bridge/internal/frontend" "github.com/ProtonMail/proton-bridge/internal/importexport" + "github.com/ProtonMail/proton-bridge/internal/updates" "github.com/ProtonMail/proton-bridge/internal/users/credentials" "github.com/ProtonMail/proton-bridge/pkg/config" "github.com/ProtonMail/proton-bridge/pkg/constants" "github.com/ProtonMail/proton-bridge/pkg/listener" "github.com/ProtonMail/proton-bridge/pkg/pmapi" - "github.com/ProtonMail/proton-bridge/pkg/updates" "github.com/allan-simon/go-singleinstance" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -47,8 +46,8 @@ var ( func main() { cmd.Main( - "ProtonMail Import/Export", - "ProtonMail Import/Export tool", + "ProtonMail Import-Export", + "ProtonMail Import-Export app", nil, run, ) @@ -66,7 +65,7 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen] // report which will not be possible if no folder can be created. That's the // only problem we will not be notified about in any way. panicHandler := &cmd.PanicHandler{ - AppName: "ProtonMail Import/Export", + AppName: "ProtonMail Import-Export", Config: cfg, Err: &contextError, } @@ -81,7 +80,7 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen] logLevel := context.GlobalString("log-level") _, _ = config.SetupLog(cfg, logLevel) - // Doesn't make sense to continue when Import/Export was invoked with wrong arguments. + // Doesn't make sense to continue when Import-Export was invoked with wrong arguments. // We should tell that to the user before we do anything else. if context.Args().First() != "" { _ = cli.ShowAppHelp(context) @@ -89,7 +88,7 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen] } // It's safe to get version JSON file even when other instance is running. - // (thus we put it before check of presence of other Import/Export instance). + // (thus we put it before check of presence of other Import-Export instance). updates := updates.NewImportExport(cfg.GetUpdateDir()) if dir := context.GlobalString("version-json"); dir != "" { @@ -97,24 +96,18 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen] return nil } - // Now we can try to proceed with starting the import/export. First we need to ensure + // Now we can try to proceed with starting the Import-Export. First we need to ensure // this is the only instance. If not, we will end and focus the existing one. lock, err := singleinstance.CreateLockFile(cfg.GetLockPath()) if err != nil { - log.Warn("Import/Export is already running") - return cli.NewExitError("Import/Export is already running.", 3) + log.Warn("Import-Export app is already running") + return cli.NewExitError("Import-Export app is already running.", 3) } defer lock.Close() //nolint[errcheck] // In case user wants to do CPU or memory profiles... if doCPUProfile := context.GlobalBool("cpu-prof"); doCPUProfile { - f, err := os.Create("cpu.pprof") - if err != nil { - log.Fatal("Could not create CPU profile: ", err) - } - if err := pprof.StartCPUProfile(f); err != nil { - log.Fatal("Could not start CPU profile: ", err) - } + cmd.StartCPUProfile() defer pprof.StopCPUProfile() } @@ -122,8 +115,8 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen] defer cmd.MakeMemoryProfile() } - // Now we initialize all Import/Export parts. - log.Debug("Initializing import/export...") + // Now we initialize all Import-Export parts. + log.Debug("Initializing import-export...") eventListener := listener.New() events.SetupEvents(eventListener) @@ -141,7 +134,7 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen] importexportInstance := importexport.New(cfg, panicHandler, eventListener, cm, credentialsStore) - // Decide about frontend mode before initializing rest of import/export. + // Decide about frontend mode before initializing rest of import-export. var frontendMode string switch { case context.GlobalBool("cli"): diff --git a/doc/importexport.md b/doc/importexport.md index e8449ced..8dd2303b 100644 --- a/doc/importexport.md +++ b/doc/importexport.md @@ -1,15 +1,15 @@ -# Import/Export +# Import-Export ## Main blocks -This is basic overview of the main import/export blocks. +This is basic overview of the main import-export blocks. ```mermaid graph LR S[ProtonMail server] U[User] - subgraph "Import/Export app" + subgraph "Import-Export app" Users Frontend["Qt / CLI"] ImportExport @@ -35,7 +35,7 @@ graph LR ## Code structure -More detailed graph of main types used in Import/Export app and connection between them. +More detailed graph of main types used in Import-Export app and connection between them. ```mermaid graph TD @@ -44,9 +44,9 @@ graph TD MBOX[MBOX] IMAP[IMAP] - subgraph "Import/Export app" - subgraph PkgUsers - subgraph PkgCredentials + subgraph "Import-Export app" + subgraph "pkg users" + subgraph "pkg credentials" CredStore[Store] Creds[Credentials] @@ -59,16 +59,16 @@ graph TD US --> U end - subgraph PkgFrontend + subgraph "pkg frontend" CLI Qt end - subgraph PkgImportExport + subgraph "pkg importExport" IE[ImportExport] end - subgraph PkgTransfer + subgraph "pkg transfer" Transfer Rules Progress diff --git a/doc/index.md b/doc/index.md index b6d3a543..4791f34a 100644 --- a/doc/index.md +++ b/doc/index.md @@ -9,6 +9,6 @@ Documentation pages in order to read for a novice: * [Communication between Bridge, Client and Server](communication.md) * [Encryption](encryption.md) -## Import/Export +## Import-Export -* [Import/Export code](importexport.md) +* [Import-Export code](importexport.md) diff --git a/internal/bridge/credits.go b/internal/bridge/credits.go index 2df79e67..0efc59ab 100644 --- a/internal/bridge/credits.go +++ b/internal/bridge/credits.go @@ -15,8 +15,8 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . -// Code generated by ./credits.sh at Wed 29 Jul 2020 10:20:09 AM CEST. DO NOT EDIT. +// Code generated by ./credits.sh at Wed Aug 12 09:33:24 CEST 2020. DO NOT EDIT. package bridge -const Credits = "github.com/0xAX/notificator;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/andybalholm/cascadia;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/go-resty/resty/v2;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/jhillyerd/enmime;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/gopenpgp/v2;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;" +const Credits = "github.com/0xAX/notificator;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/ProtonMail/gopenpgp/v2;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/andybalholm/cascadia;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-delve/delve;github.com/go-resty/resty/v2;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/jhillyerd/enmime;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/psampaz/go-mod-outdated;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;" diff --git a/internal/cmd/memory_profile.go b/internal/cmd/profiles.go similarity index 75% rename from internal/cmd/memory_profile.go rename to internal/cmd/profiles.go index 5b09bf54..71fab3c5 100644 --- a/internal/cmd/memory_profile.go +++ b/internal/cmd/profiles.go @@ -24,12 +24,23 @@ import ( "runtime/pprof" ) +// StartCPUProfile starts CPU pprof. +func StartCPUProfile() { + f, err := os.Create("./cpu.pprof") + if err != nil { + log.Fatal("Could not create CPU profile: ", err) + } + if err := pprof.StartCPUProfile(f); err != nil { + log.Fatal("Could not start CPU profile: ", err) + } +} + // MakeMemoryProfile generates memory pprof. func MakeMemoryProfile() { name := "./mem.pprof" f, err := os.Create(name) if err != nil { - log.Error("Could not create memory profile: ", err) + log.Fatal("Could not create memory profile: ", err) } if abs, err := filepath.Abs(name); err == nil { name = abs @@ -37,7 +48,7 @@ func MakeMemoryProfile() { log.Info("Writing memory profile to ", name) runtime.GC() // get up-to-date statistics if err := pprof.WriteHeapProfile(f); err != nil { - log.Error("Could not write memory profile: ", err) + log.Fatal("Could not write memory profile: ", err) } _ = f.Close() } diff --git a/internal/cmd/version_file.go b/internal/cmd/version_file.go index fe0e8e9c..bb832254 100644 --- a/internal/cmd/version_file.go +++ b/internal/cmd/version_file.go @@ -17,7 +17,7 @@ package cmd -import "github.com/ProtonMail/proton-bridge/pkg/updates" +import "github.com/ProtonMail/proton-bridge/internal/updates" // GenerateVersionFiles writes a JSON file with details about current build. // Those files are used for upgrading the app. diff --git a/internal/frontend/cli-ie/account_utils.go b/internal/frontend/cli-ie/account_utils.go index d016ce26..0988fcaa 100644 --- a/internal/frontend/cli-ie/account_utils.go +++ b/internal/frontend/cli-ie/account_utils.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . -package cli +package cliie import ( "fmt" diff --git a/internal/frontend/cli-ie/accounts.go b/internal/frontend/cli-ie/accounts.go index ae3764f3..b0d25130 100644 --- a/internal/frontend/cli-ie/accounts.go +++ b/internal/frontend/cli-ie/accounts.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . -package cli +package cliie import ( "strings" diff --git a/internal/frontend/cli-ie/frontend.go b/internal/frontend/cli-ie/frontend.go index cf769fe6..3388a679 100644 --- a/internal/frontend/cli-ie/frontend.go +++ b/internal/frontend/cli-ie/frontend.go @@ -15,8 +15,8 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . -// Package cli provides CLI interface of the Import/Export. -package cli +// Package cliie provides CLI interface of the Import-Export app. +package cliie import ( "github.com/ProtonMail/proton-bridge/internal/events" @@ -68,7 +68,7 @@ func New( //nolint[funlen] Aliases: []string{"cl"}, } clearCmd.AddCmd(&ishell.Cmd{Name: "accounts", - Help: "remove all accounts from keychain. (aliases: k, keychain)", + Help: "remove all accounts from keychain. (aliases: a, k, keychain)", Aliases: []string{"a", "k", "keychain"}, Func: fe.deleteAccounts, }) @@ -77,7 +77,7 @@ func New( //nolint[funlen] // Check commands. checkCmd := &ishell.Cmd{Name: "check", Help: "check internet connection or new version."} checkCmd.AddCmd(&ishell.Cmd{Name: "updates", - Help: "check for Import/Export updates. (aliases: u, v, version)", + Help: "check for Import-Export updates. (aliases: u, v, version)", Aliases: []string{"u", "version", "v"}, Func: fe.checkUpdates, }) @@ -134,7 +134,7 @@ func New( //nolint[funlen] Completer: fe.completeUsernames, }) - // Import/Export commands. + // Import-Export commands. importCmd := &ishell.Cmd{Name: "import", Help: "import messages. (alias: imp)", Aliases: []string{"imp"}, @@ -167,7 +167,7 @@ func New( //nolint[funlen] // System commands. fe.AddCmd(&ishell.Cmd{Name: "restart", - Help: "restart the import/export.", + Help: "restart the Import-Export app.", Func: fe.restart, }) @@ -190,7 +190,7 @@ func (f *frontendCLI) watchEvents() { for { select { case errorDetails := <-errorCh: - f.Println("Import/Export failed:", errorDetails) + f.Println("Import-Export failed:", errorDetails) case <-internetOffCh: f.notifyInternetOff() case <-internetOnCh: @@ -228,9 +228,9 @@ func (f *frontendCLI) Loop(credentialsError error) error { } f.Print(` -Welcome to ProtonMail Import/Export interactive shell +Welcome to ProtonMail Import-Export interactive shell -WARNING: CLI is experimental feature and does not cover all functionality yet. +WARNING: The CLI is an experimental feature and does not yet cover all functionality. `) f.Run() return nil diff --git a/internal/frontend/cli-ie/importexport.go b/internal/frontend/cli-ie/importexport.go index 05db81bf..fe89d7df 100644 --- a/internal/frontend/cli-ie/importexport.go +++ b/internal/frontend/cli-ie/importexport.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . -package cli +package cliie import ( "fmt" diff --git a/internal/frontend/cli-ie/system.go b/internal/frontend/cli-ie/system.go index 2df6574e..103b4511 100644 --- a/internal/frontend/cli-ie/system.go +++ b/internal/frontend/cli-ie/system.go @@ -15,19 +15,15 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . -package cli +package cliie import ( "github.com/abiosoft/ishell" ) -var ( - currentPort = "" //nolint[gochecknoglobals] -) - func (f *frontendCLI) restart(c *ishell.Context) { - if f.yesNoQuestion("Are you sure you want to restart the Import/Export") { - f.Println("Restarting Import/Export...") + if f.yesNoQuestion("Are you sure you want to restart the Import-Export") { + f.Println("Restarting the Import-Export app...") f.appRestart = true f.Stop() } @@ -37,7 +33,7 @@ func (f *frontendCLI) checkInternetConnection(c *ishell.Context) { if f.ie.CheckConnection() == nil { f.Println("Internet connection is available.") } else { - f.Println("Can not contact the server, please check you internet connection.") + f.Println("Can not contact the server, please check your internet connection.") } } @@ -46,5 +42,5 @@ func (f *frontendCLI) printLogDir(c *ishell.Context) { } func (f *frontendCLI) printManual(c *ishell.Context) { - f.Println("More instructions about the Import/Export can be found at\n\n https://protonmail.com/support/categories/import-export/") + f.Println("More instructions about the Import-Export app can be found at\n\n https://protonmail.com/support/categories/import-export/") } diff --git a/internal/frontend/cli-ie/updates.go b/internal/frontend/cli-ie/updates.go index 62332dc8..cabebd79 100644 --- a/internal/frontend/cli-ie/updates.go +++ b/internal/frontend/cli-ie/updates.go @@ -15,13 +15,13 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . -package cli +package cliie import ( "strings" "github.com/ProtonMail/proton-bridge/internal/importexport" - "github.com/ProtonMail/proton-bridge/pkg/updates" + "github.com/ProtonMail/proton-bridge/internal/updates" "github.com/abiosoft/ishell" ) @@ -47,7 +47,7 @@ func (f *frontendCLI) printLocalReleaseNotes(c *ishell.Context) { } func (f *frontendCLI) printReleaseNotes(versionInfo updates.VersionInfo) { - f.Println(bold("ProtonMail Import/Export "+versionInfo.Version), "\n") + f.Println(bold("ProtonMail Import-Export "+versionInfo.Version), "\n") if versionInfo.ReleaseNotes != "" { f.Println(bold("Release Notes")) f.Println(versionInfo.ReleaseNotes) diff --git a/internal/frontend/cli-ie/utils.go b/internal/frontend/cli-ie/utils.go index 6f01d2fe..b2de679a 100644 --- a/internal/frontend/cli-ie/utils.go +++ b/internal/frontend/cli-ie/utils.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . -package cli +package cliie import ( "strings" @@ -98,7 +98,7 @@ func (f *frontendCLI) notifyNeedUpgrade() { func (f *frontendCLI) notifyCredentialsError() { // Print in 80-column width. - f.Println("ProtonMail Import/Export is not able to detect a supported password manager") + f.Println("ProtonMail Import-Export is not able to detect a supported password manager") f.Println("(pass, gnome-keyring). Please install and set up a supported password manager") f.Println("and restart the application.") } @@ -109,7 +109,7 @@ func (f *frontendCLI) notifyCertIssue() { be insecure. Description: -ProtonMail Import/Export was not able to establish a secure connection to Proton +ProtonMail Import-Export was not able to establish a secure connection to Proton servers due to a TLS certificate error. This means your connection may potentially be insecure and susceptible to monitoring by third parties. diff --git a/internal/frontend/cli/frontend.go b/internal/frontend/cli/frontend.go index 0df33e7b..82c0dbc6 100644 --- a/internal/frontend/cli/frontend.go +++ b/internal/frontend/cli/frontend.go @@ -76,7 +76,7 @@ func New( //nolint[funlen] Func: fe.deleteCache, }) clearCmd.AddCmd(&ishell.Cmd{Name: "accounts", - Help: "remove all accounts from keychain. (aliases: k, keychain)", + Help: "remove all accounts from keychain. (aliases: a, k, keychain)", Aliases: []string{"a", "k", "keychain"}, Func: fe.deleteAccounts, }) diff --git a/internal/frontend/cli/system.go b/internal/frontend/cli/system.go index a32dcc9c..fe01fa4a 100644 --- a/internal/frontend/cli/system.go +++ b/internal/frontend/cli/system.go @@ -43,7 +43,7 @@ func (f *frontendCLI) checkInternetConnection(c *ishell.Context) { if f.bridge.CheckConnection() == nil { f.Println("Internet connection is available.") } else { - f.Println("Can not contact the server, please check you internet connection.") + f.Println("Can not contact the server, please check your internet connection.") } } diff --git a/internal/frontend/cli/updates.go b/internal/frontend/cli/updates.go index b30da468..44951fec 100644 --- a/internal/frontend/cli/updates.go +++ b/internal/frontend/cli/updates.go @@ -21,7 +21,7 @@ import ( "strings" "github.com/ProtonMail/proton-bridge/internal/bridge" - "github.com/ProtonMail/proton-bridge/pkg/updates" + "github.com/ProtonMail/proton-bridge/internal/updates" "github.com/abiosoft/ishell" ) diff --git a/internal/frontend/qml/ImportExportUI/DialogImport.qml b/internal/frontend/qml/ImportExportUI/DialogImport.qml index c4fd4c1b..2dce1ea8 100644 --- a/internal/frontend/qml/ImportExportUI/DialogImport.qml +++ b/internal/frontend/qml/ImportExportUI/DialogImport.qml @@ -458,7 +458,7 @@ Dialog { if (progressbarImport.isFinished) return qsTr("Import finished","todo") if ( go.progressDescription == gui.enums.progressInit || - (go.progress == 0 && go.description=="") + (go.progress == 0 && go.progressDescription=="") ) return qsTr("Estimating the total number of messages","todo") if ( go.progressDescription == gui.enums.progressLooping diff --git a/internal/frontend/qml/ImportExportUI/ImportDelegate.qml b/internal/frontend/qml/ImportExportUI/ImportDelegate.qml index aa916186..f59e6d7b 100644 --- a/internal/frontend/qml/ImportExportUI/ImportDelegate.qml +++ b/internal/frontend/qml/ImportExportUI/ImportDelegate.qml @@ -43,6 +43,8 @@ Rectangle { property string lastTargetFolder: "6" // Archive property string lastTargetLabels: "" // no flag by default + property string sourceID : mboxID + property string sourceName : name Rectangle { id: line @@ -71,7 +73,7 @@ Rectangle { Text { id: folderIcon - text : gui.folderIcon(name, gui.enums.folderTypeFolder) + text : gui.folderIcon(root.sourceName, gui.enums.folderTypeFolder) anchors.verticalCenter : parent.verticalCenter color: root.isSourceSelected ? Style.main.text : Style.main.textDisabled font { @@ -81,7 +83,7 @@ Rectangle { } Text { - text : name + text : root.sourceName width: nameWidth elide: Text.ElideRight anchors.verticalCenter : parent.verticalCenter @@ -102,8 +104,8 @@ Rectangle { SelectFolderMenu { id: selectFolder - sourceID: mboxID - targets: transferRules.targetFolders(mboxID) + sourceID: root.sourceID + targets: transferRules.targetFolders(root.sourceID) width: nameWidth anchors.verticalCenter : parent.verticalCenter enabled: root.isSourceSelected @@ -112,8 +114,8 @@ Rectangle { } SelectLabelsMenu { - sourceID: mboxID - targets: transferRules.targetLabels(mboxID) + sourceID: root.sourceID + targets: transferRules.targetLabels(root.sourceID) width: nameWidth anchors.verticalCenter : parent.verticalCenter enabled: root.isSourceSelected @@ -130,7 +132,7 @@ Rectangle { DateRangeMenu { id: dateRangeMenu - sourceID: mboxID + sourceID: root.sourceID sourceFromDate: fromDate sourceToDate: toDate @@ -143,10 +145,10 @@ Rectangle { function importToFolder(newTargetID) { - transferRules.addTargetID(mboxID,newTargetID) + transferRules.addTargetID(root.sourceID,newTargetID) } function toggleImport() { - transferRules.setIsRuleActive(mboxID, !root.isSourceSelected) + transferRules.setIsRuleActive(root.sourceID, !root.isSourceSelected) } } diff --git a/internal/frontend/qml/ImportExportUI/ImportStructure.qml b/internal/frontend/qml/ImportExportUI/ImportStructure.qml index 17956039..9628b7d7 100644 --- a/internal/frontend/qml/ImportExportUI/ImportStructure.qml +++ b/internal/frontend/qml/ImportExportUI/ImportStructure.qml @@ -110,7 +110,7 @@ Rectangle { left: parent.left verticalCenter: parent.verticalCenter leftMargin: { - if (listview.currentIndex<0) return 0 + if (listview.currentItem === null) return 0 else return listview.currentItem.leftMargin1 } } diff --git a/internal/frontend/qml/ImportExportUI/MainWindow.qml b/internal/frontend/qml/ImportExportUI/MainWindow.qml index 2531c803..48e5353f 100644 --- a/internal/frontend/qml/ImportExportUI/MainWindow.qml +++ b/internal/frontend/qml/ImportExportUI/MainWindow.qml @@ -112,7 +112,7 @@ Window { rightMargin: innerWindowBorder } model: [ - { "title" : qsTr("Import/Export" , "title of tab that shows account list" ), "iconText": Style.fa.home }, + { "title" : qsTr("Import-Export" , "title of tab that shows account list" ), "iconText": Style.fa.home }, { "title" : qsTr("Settings" , "title of tab that allows user to change settings" ), "iconText": Style.fa.cogs }, { "title" : qsTr("Help" , "title of tab that shows the help menu" ), "iconText": Style.fa.life_ring } ] @@ -381,8 +381,9 @@ Window { onClickedNo: popupMessage.hide() onClickedOkay: popupMessage.hide() + onClickedCancel: popupMessage.hide() onClickedYes: { - if (popupMessage.message == gui.areYouSureYouWantToQuit) Qt.quit() + if (popupMessage.text == gui.areYouSureYouWantToQuit) Qt.quit() } } @@ -461,8 +462,9 @@ Window { (dialogExport.visible && dialogExport.currentIndex == 2 && go.progress!=1) ) { popupMessage.buttonOkay .visible = false - popupMessage.buttonNo .visible = true - popupMessage.buttonYes .visible = true + popupMessage.buttonYes .visible = false + popupMessage.buttonQuit .visible = true + popupMessage.buttonCancel .visible = true popupMessage.show ( gui.areYouSureYouWantToQuit ) return } diff --git a/internal/frontend/qml/ImportExportUI/SelectFolderMenu.qml b/internal/frontend/qml/ImportExportUI/SelectFolderMenu.qml index 2dc3b20d..5a1e7c20 100644 --- a/internal/frontend/qml/ImportExportUI/SelectFolderMenu.qml +++ b/internal/frontend/qml/ImportExportUI/SelectFolderMenu.qml @@ -58,7 +58,6 @@ ComboBox { } displayText: { - console.log("Target Menu current", view.currentItem, view.currentIndex) if (view.currentIndex >= 0) { if (!root.isFolderType) return Style.fa.tags + " " + qsTr("Add/Remove labels") diff --git a/internal/frontend/qml/ProtonUI/PopupMessage.qml b/internal/frontend/qml/ProtonUI/PopupMessage.qml index 5c2f32ae..ffaa4556 100644 --- a/internal/frontend/qml/ProtonUI/PopupMessage.qml +++ b/internal/frontend/qml/ProtonUI/PopupMessage.qml @@ -25,6 +25,7 @@ Rectangle { color: Style.transparent property alias text : message.text property alias checkbox : checkbox + property alias buttonQuit : buttonQuit property alias buttonOkay : buttonOkay property alias buttonYes : buttonYes property alias buttonNo : buttonNo @@ -89,13 +90,13 @@ Rectangle { spacing: Style.dialog.spacing anchors.horizontalCenter : parent.horizontalCenter + ButtonRounded { id : buttonQuit ; text : qsTr ( "Stop & quit", "" ) ; onClicked : root.clickedYes ( ) ; visible : false ; isOpaque : true ; color_main : Style.errorDialog.text ; color_minor : Style.dialog.textBlue ; } ButtonRounded { id : buttonNo ; text : qsTr ( "No" , "Button No" ) ; onClicked : root.clickedNo ( ) ; visible : false ; isOpaque : false ; color_main : Style.errorDialog.text ; color_minor : Style.transparent ; } ButtonRounded { id : buttonYes ; text : qsTr ( "Yes" , "Button Yes" ) ; onClicked : root.clickedYes ( ) ; visible : false ; isOpaque : true ; color_main : Style.errorDialog.text ; color_minor : Style.dialog.textBlue ; } ButtonRounded { id : buttonRetry ; text : qsTr ( "Retry" , "Button Retry" ) ; onClicked : root.clickedRetry ( ) ; visible : false ; isOpaque : false ; color_main : Style.errorDialog.text ; color_minor : Style.transparent ; } ButtonRounded { id : buttonSkip ; text : qsTr ( "Skip" , "Button Skip" ) ; onClicked : root.clickedSkip ( ) ; visible : false ; isOpaque : false ; color_main : Style.errorDialog.text ; color_minor : Style.transparent ; } ButtonRounded { id : buttonCancel ; text : qsTr ( "Cancel" , "Button Cancel" ) ; onClicked : root.clickedCancel ( ) ; visible : false ; isOpaque : true ; color_main : Style.errorDialog.text ; color_minor : Style.dialog.textBlue ; } ButtonRounded { id : buttonOkay ; text : qsTr ( "Okay" , "Button Okay" ) ; onClicked : root.clickedOkay ( ) ; visible : true ; isOpaque : true ; color_main : Style.errorDialog.text ; color_minor : Style.dialog.textBlue ; } - } } } diff --git a/internal/frontend/qml/tst_GuiIE.qml b/internal/frontend/qml/tst_GuiIE.qml index c96f4ce1..48dcc460 100644 --- a/internal/frontend/qml/tst_GuiIE.qml +++ b/internal/frontend/qml/tst_GuiIE.qml @@ -99,7 +99,7 @@ Window { id: buttons ListElement { title : "Show window" } - ListElement { title : "Logout cuthix" } + ListElement { title : "Logout" } ListElement { title : "Internet on" } ListElement { title : "Internet off" } ListElement { title : "Macos" } @@ -143,8 +143,8 @@ Window { case "Show window" : go.showWindow(); break; - case "Logout cuthix" : - go.checkLoggedOut("cuthix"); + case "Logout" : + go.checkLoggedOut("ie"); break; case "Internet on" : go.setConnectionStatus(true); @@ -223,10 +223,10 @@ Window { ListModel{ id: accountsModel - ListElement{ account : "cuthix" ; status : "connected"; isExpanded: false; isCombinedAddressMode: false; hostname : "127.0.0.1"; password : "ZI9tKp+ryaxmbpn2E12"; security : "StarTLS"; portSMTP : 1025; portIMAP : 1143; aliases : "cuto@pm.com;jaku@pm.com;DoYouKnowAboutAMovieCalledTheHorriblySlowMurderWithExtremelyInefficientWeapon@thatYouCanFindForExampleOnyoutube.com" } - ListElement{ account : "exteremelongnamewhichmustbeeladedinthemiddleoftheaddress@protonmail.com" ; status : "connected"; isExpanded: true; isCombinedAddressMode: true; hostname : "127.0.0.1"; password : "ZI9tKp+ryaxmbpn2E12"; security : "StarTLS"; portSMTP : 1025; portIMAP : 1143; aliases : "cuto@pm.com;jaku@pm.com;hu@hu.hu" } - ListElement{ account : "cuthix2@protonmail.com" ; status : "disconnected"; isExpanded: false; isCombinedAddressMode: false; hostname : "127.0.0.1"; password : "ZI9tKp+ryaxmbpn2E12"; security : "StarTLS"; portSMTP : 1025; portIMAP : 1143; aliases : "cuto@pm.com;jaku@pm.com;hu@hu.hu" } - ListElement{ account : "many@protonmail.com" ; status : "connected"; isExpanded: true; isCombinedAddressMode: true; hostname : "127.0.0.1"; password : "ZI9tKp+ryaxmbpn2E12"; security : "StarTLS"; portSMTP : 1025; portIMAP : 1143; aliases : "cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;cuto@pm.com;jaku@pm.com;hu@hu.hu;"} + ListElement{ account : "ie" ; status : "connected"; isExpanded: false; isCombinedAddressMode: false; hostname : "127.0.0.1"; password : "ZI9tKp+ryaxmbpn2E12"; security : "StarTLS"; portSMTP : 1025; portIMAP : 1143; aliases : "ie@pm.com;jaku@pm.com;DoYouKnowAboutAMovieCalledTheHorriblySlowMurderWithExtremelyInefficientWeapon@thatYouCanFindForExampleOnyoutube.com" } + ListElement{ account : "exteremelongnamewhichmustbeeladedinthemiddleoftheaddress@protonmail.com" ; status : "connected"; isExpanded: true; isCombinedAddressMode: true; hostname : "127.0.0.1"; password : "ZI9tKp+ryaxmbpn2E12"; security : "StarTLS"; portSMTP : 1025; portIMAP : 1143; aliases : "ie@pm.com;jaku@pm.com;hu@hu.hu" } + ListElement{ account : "ie2@protonmail.com" ; status : "disconnected"; isExpanded: false; isCombinedAddressMode: false; hostname : "127.0.0.1"; password : "ZI9tKp+ryaxmbpn2E12"; security : "StarTLS"; portSMTP : 1025; portIMAP : 1143; aliases : "ie@pm.com;jaku@pm.com;hu@hu.hu" } + ListElement{ account : "many@protonmail.com" ; status : "connected"; isExpanded: true; isCombinedAddressMode: true; hostname : "127.0.0.1"; password : "ZI9tKp+ryaxmbpn2E12"; security : "StarTLS"; portSMTP : 1025; portIMAP : 1143; aliases : "ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;ie@pm.com;jaku@pm.com;hu@hu.hu;"} } ListModel{ @@ -830,9 +830,9 @@ Window { property string bugNotSent property string bugReportSent - property string programTitle : "ProtonMail Import/Export Tool" + property string programTitle : "ProtonMail Import-Export App" property string newversion : "q0.1.0" - property string landingPage : "https://jakub.cuth.sk/bridge" + property string landingPage : "https://landing.page" property string changelog : "• Lorem ipsum dolor sit amet\n• consetetur sadipscing elitr,\n• sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat,\n• sed diam voluptua.\n• At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." //property string changelog : "" property string bugfixes : "• lorem ipsum dolor sit amet;• consetetur sadipscing elitr;• sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat;• sed diam voluptua;• at vero eos et accusam et justo duo dolores et ea rebum;• stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet" diff --git a/internal/frontend/qt-common/common.go b/internal/frontend/qt-common/common.go index 644eefb6..d0143d48 100644 --- a/internal/frontend/qt-common/common.go +++ b/internal/frontend/qt-common/common.go @@ -103,16 +103,6 @@ func PauseLong() { time.Sleep(3 * time.Second) } -func ParsePMAPIError(err error, code int) error { - /* - if err == pmapi.ErrAPINotReachable { - code = ErrNoInternet - } - return errors.NewFromError(code, err) - */ - return nil -} - // FIXME: Not working in test... func WaitForEnter() { log.Print("Press 'Enter' to continue...") diff --git a/internal/frontend/qt-ie/export.go b/internal/frontend/qt-ie/export.go index e851ff02..e53922e1 100644 --- a/internal/frontend/qt-ie/export.go +++ b/internal/frontend/qt-ie/export.go @@ -47,6 +47,18 @@ func (f *FrontendQt) LoadStructureForExport(addressOrID string) { return } + // Export has only one option to set time limits--by global time range. + // In case user changes file or because of some bug global time is saved + // to all rules, let's clear it, because there is no way to show it in + // GUI and user would be confused and see it does not work at all. + for _, rule := range f.transfer.GetRules() { + isActive := rule.Active + f.transfer.SetRule(rule.SourceMailbox, rule.TargetMailboxes, 0, 0) + if !isActive { + f.transfer.UnsetRule(rule.SourceMailbox) + } + } + f.TransferRules.setTransfer(f.transfer) } @@ -65,55 +77,4 @@ func (f *FrontendQt) StartExport(rootPath, login, fileType string, attachEncrypt f.transfer.SetSkipEncryptedMessages(!attachEncryptedBody) progress := f.transfer.Start() f.setProgressManager(progress) - - /* - TODO - f.Qml.SetProgress(0.0) - f.Qml.SetProgressDescription(backend.ProgressInit) - f.Qml.SetTotal(0) - - settings := backend.ExportSettings{ - FilePath: fpath, - Login: login, - AttachEncryptedBody: attachEncryptedBody, - DateBegin: 0, - DateEnd: 0, - Labels: make(map[string]string), - } - - if fileType == "EML" { - settings.FileTypeID = backend.EMLFormat - } else if fileType == "MBOX" { - settings.FileTypeID = backend.MBOXFormat - } else { - log.Errorln("Wrong file format:", fileType) - return - } - - username, _, err := backend.ExtractUsername(login) - if err != nil { - log.Error("qtfrontend: cannot retrieve username from alias: ", err) - return - } - - settings.User, err = backend.ExtractCurrentUser(username) - if err != nil && !errors.IsCode(err, errors.ErrUnlockUser) { - return - } - - for _, entity := range f.PMStructure.entities { - if entity.IsFolderSelected { - settings.Labels[entity.FolderName] = entity.FolderId - } - } - - settings.DateBegin = f.PMStructure.GlobalOptions.FromDate - settings.DateEnd = f.PMStructure.GlobalOptions.ToDate - - settings.PM = backend.NewProcessManager() - f.setHandlers(settings.PM) - - log.Debugln("start export", settings.FilePath) - go backend.Export(f.panicHandler, settings) - */ } diff --git a/internal/frontend/qt-ie/folder_functions.go b/internal/frontend/qt-ie/folder_functions.go index 7c156a72..851d770d 100644 --- a/internal/frontend/qt-ie/folder_functions.go +++ b/internal/frontend/qt-ie/folder_functions.go @@ -59,10 +59,6 @@ func getTargetHashes(mboxes []transfer.Mailbox) (targetFolderID, targetLabelIDs return } -func isSystemMailbox(mbox transfer.Mailbox) bool { - return pmapi.IsSystemLabel(mbox.ID) -} - func newFolderInfo(mbox transfer.Mailbox, rule *transfer.Rule) *FolderInfo { targetFolderID, targetLabelIDs := getTargetHashes(rule.TargetMailboxes) @@ -77,7 +73,7 @@ func newFolderInfo(mbox transfer.Mailbox, rule *transfer.Rule) *FolderInfo { } entry.FolderType = FolderTypeSystem - if !isSystemMailbox(mbox) { + if !pmapi.IsSystemLabel(mbox.ID) { if mbox.IsExclusive { entry.FolderType = FolderTypeFolder } else { @@ -112,7 +108,7 @@ func (s *FolderStructure) saveRule(info *FolderInfo) error { return s.transfer.SetRule(sourceMbox, targetMboxes, info.FromDate, info.ToDate) } -func (s *FolderInfo) updateTgtLblIDs(targetLabelsSet map[string]struct{}) { +func (s *FolderInfo) updateTargetLabelIDs(targetLabelsSet map[string]struct{}) { targets := []string{} for key := range targetLabelsSet { targets = append(targets, key) @@ -120,17 +116,13 @@ func (s *FolderInfo) updateTgtLblIDs(targetLabelsSet map[string]struct{}) { s.TargetLabelIDs = strings.Join(targets, ";") } -func (s *FolderInfo) clearTgtLblIDs() { - s.TargetLabelIDs = "" -} - func (s *FolderInfo) AddTargetLabel(targetID string) { if targetID == "" { return } targetLabelsSet := s.getSetOfLabels() targetLabelsSet[targetID] = struct{}{} - s.updateTgtLblIDs(targetLabelsSet) + s.updateTargetLabelIDs(targetLabelsSet) } func (s *FolderInfo) RemoveTargetLabel(targetID string) { @@ -139,7 +131,7 @@ func (s *FolderInfo) RemoveTargetLabel(targetID string) { } targetLabelsSet := s.getSetOfLabels() delete(targetLabelsSet, targetID) - s.updateTgtLblIDs(targetLabelsSet) + s.updateTargetLabelIDs(targetLabelsSet) } func (s *FolderInfo) IsType(askType string) bool { @@ -387,7 +379,7 @@ func (s *FolderStructure) setTargetFolderID(id, target string) { s.changedEntityRole(i, i, TargetFolderID) if target == "" { // do not import before := info.TargetLabelIDs - info.clearTgtLblIDs() + info.TargetLabelIDs = "" if err := s.saveRule(info); err != nil { info.TargetLabelIDs = before log.WithError(err).WithField("id", id).WithField("target", target).Error("Cannot set target") diff --git a/internal/frontend/qt-ie/frontend.go b/internal/frontend/qt-ie/frontend.go index 10599904..32971fa7 100644 --- a/internal/frontend/qt-ie/frontend.go +++ b/internal/frontend/qt-ie/frontend.go @@ -29,9 +29,9 @@ import ( qtcommon "github.com/ProtonMail/proton-bridge/internal/frontend/qt-common" "github.com/ProtonMail/proton-bridge/internal/frontend/types" "github.com/ProtonMail/proton-bridge/internal/transfer" + "github.com/ProtonMail/proton-bridge/internal/updates" "github.com/ProtonMail/proton-bridge/pkg/config" "github.com/ProtonMail/proton-bridge/pkg/listener" - "github.com/ProtonMail/proton-bridge/pkg/updates" "github.com/therecipe/qt/core" "github.com/therecipe/qt/gui" @@ -185,7 +185,7 @@ func (f *FrontendQt) qtSetupQmlAndStructures() { f.View.Load(core.NewQUrl3("qrc:/uiie.qml", 0)) // TODO set the first start flag - log.Error("Get FirstStart: Not implemented") + //log.Error("Get FirstStart: Not implemented") //if prefs.Get(prefs.FirstStart) == "true" { if false { f.Qml.SetIsFirstStart(true) @@ -226,7 +226,6 @@ func (f *FrontendQt) QtExecute(Procedure func(*FrontendQt) error) error { return err } log.Debug("Closing...") - log.Error("Set FirstStart: Not implemented") //prefs.Set(prefs.FirstStart, "false") return nil } @@ -318,27 +317,31 @@ func (f *FrontendQt) setProgressManager(progress *transfer.Progress) { f.Qml.ConnectCancelProcess(func() { progress.Stop() }) + f.Qml.SetProgress(0) go func() { + log.Trace("Start reading updates") defer func() { + log.Trace("Finishing reading updates") f.Qml.DisconnectPauseProcess() f.Qml.DisconnectResumeProcess() f.Qml.DisconnectCancelProcess() f.Qml.SetProgress(1) + f.progress = nil + f.ErrorList.Progress = nil }() - //TODO get log file (in old code it was here, but this is ugly place probably somewhere else) updates := progress.GetUpdateChannel() for range updates { if progress.IsStopped() { break } failed, imported, _, _, total := progress.GetCounts() - if total != 0 { // udate total + if total != 0 { f.Qml.SetTotal(int(total)) } f.Qml.SetProgressFails(int(failed)) - f.Qml.SetProgressDescription(progress.PauseReason()) // TODO add description when changing folders? + f.Qml.SetProgressDescription(progress.PauseReason()) if total > 0 { newProgress := float32(imported+failed) / float32(total) if newProgress >= 0 && newProgress != f.Qml.Progress() { @@ -436,7 +439,7 @@ func (f *FrontendQt) getLocalVersionInfo() { // LeastUsedColor is intended to return color for creating a new inbox or label. func (f *FrontendQt) leastUsedColor() string { if f.transfer == nil { - log.Errorln("Getting least used color before transfer exist.") + log.Warnln("Getting least used color before transfer exist.") return "#7272a7" } diff --git a/internal/frontend/qt-ie/import.go b/internal/frontend/qt-ie/import.go index 6e80dda9..c26bad32 100644 --- a/internal/frontend/qt-ie/import.go +++ b/internal/frontend/qt-ie/import.go @@ -74,6 +74,8 @@ func (f *FrontendQt) loadStructuresForImport() error { } func (f *FrontendQt) StartImport(email string) { // TODO email not needed + log.Trace("Starting import") + f.Qml.SetProgressDescription("init") // TODO use const f.Qml.SetProgressFails(0) f.Qml.SetProgress(0.0) diff --git a/internal/frontend/qt-ie/mbox.go b/internal/frontend/qt-ie/mbox.go index 7f791c89..9fe33166 100644 --- a/internal/frontend/qt-ie/mbox.go +++ b/internal/frontend/qt-ie/mbox.go @@ -55,8 +55,8 @@ func newMboxList(t *TransferRules, rule *transfer.Rule, containsFolders bool) *M m.log = log. WithField("rule", m.rule.SourceMailbox.Hash()). WithField("folders", m.containsFolders) + m.updateSelectedIndex() m.EndResetModel() - m.itemsChanged(rule) return m } @@ -71,11 +71,6 @@ func (m *MboxList) rowCount(index *core.QModelIndex) int { } func (m *MboxList) roleNames() map[int]*core.QByteArray { - m.log. - WithField("isActive", MboxIsActive). - WithField("id", MboxID). - WithField("color", MboxColor). - Debug("role names") return map[int]*core.QByteArray{ MboxIsActive: qtcommon.NewQByteArrayFromString("isActive"), MboxID: qtcommon.NewQByteArrayFromString("mboxID"), @@ -88,17 +83,17 @@ func (m *MboxList) roleNames() map[int]*core.QByteArray { func (m *MboxList) data(index *core.QModelIndex, role int) *core.QVariant { allTargets := m.targetMailboxes() - i, valid := index.Row(), index.IsValid() - l := m.log.WithField("row", i).WithField("role", role) - l.Trace("called data()") + i := index.Row() + log := m.log.WithField("row", i).WithField("role", role) + log.Trace("Mbox data") - if !valid || i >= len(allTargets) { - l.WithField("row", i).Warning("Invalid index") + if i >= len(allTargets) { + log.Warning("Invalid index") return core.NewQVariant() } if m.transfer == nil { - l.Warning("Requested mbox list data before transfer is connected") + log.Warning("Requested mbox list data before transfer is connected") return qtcommon.NewQVariantString("") } @@ -131,7 +126,7 @@ func (m *MboxList) data(index *core.QModelIndex, role int) *core.QVariant { return qtcommon.NewQVariantString(mbox.Color) default: - l.Error("Requested mbox list data with unknown role") + log.Error("Requested mbox list data with unknown role") return qtcommon.NewQVariantString("") } } @@ -161,11 +156,10 @@ func (m *MboxList) filter(mailboxes []transfer.Mailbox) (filtered []transfer.Mai func (m *MboxList) itemsChanged(rule *transfer.Rule) { m.rule = rule allTargets := m.targetMailboxes() - l := m.log.WithField("count", len(allTargets)) - l.Trace("called itemChanged()") - defer func() { - l.WithField("selected", m.SelectedIndex()).Trace("index updated") - }() + + m.log.WithField("count", len(allTargets)).Trace("Mbox items changed") + + m.updateSelectedIndex() // NOTE: Be careful with indices: If they are invalid the DataChanged // signal will not be sent to QML e.g. `end == rowCount - 1` @@ -175,7 +169,10 @@ func (m *MboxList) itemsChanged(rule *transfer.Rule) { changedRoles := []int{MboxIsActive} m.DataChanged(begin, end, changedRoles) } +} +func (m *MboxList) updateSelectedIndex() { + allTargets := m.targetMailboxes() for index, targetMailbox := range allTargets { for _, selectedTarget := range m.rule.TargetMailboxes { if targetMailbox.Hash() == selectedTarget.Hash() { diff --git a/internal/frontend/qt-common/path_status.go b/internal/frontend/qt-ie/path_status.go similarity index 98% rename from internal/frontend/qt-common/path_status.go rename to internal/frontend/qt-ie/path_status.go index bf76d6d8..0216cb56 100644 --- a/internal/frontend/qt-common/path_status.go +++ b/internal/frontend/qt-ie/path_status.go @@ -15,7 +15,9 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . -package qtcommon +// +build !nogui + +package qtie import ( "io/ioutil" diff --git a/internal/frontend/qt-ie/transfer_rules.go b/internal/frontend/qt-ie/transfer_rules.go index 8b88f94e..48719a79 100644 --- a/internal/frontend/qt-ie/transfer_rules.go +++ b/internal/frontend/qt-ie/transfer_rules.go @@ -44,6 +44,7 @@ type TransferRules struct { _ func(sourceID string, targetID string) `slot:"addTargetID,auto"` _ func(sourceID string, targetID string) `slot:"removeTargetID,auto"` + // globalFromDate and globalToDate is just default value for GUI, always zero. _ int `property:"globalFromDate"` _ int `property:"globalToDate"` _ bool `property:"isLabelGroupSelected"` @@ -90,21 +91,23 @@ func (t *TransferRules) roleNames() map[int]*core.QByteArray { } func (t *TransferRules) data(index *core.QModelIndex, role int) *core.QVariant { - i, valid := index.Row(), index.IsValid() - - if !valid || i >= t.rowCount(index) { - log.WithField("row", i).Warning("Invalid index") - return core.NewQVariant() - } + i := index.Row() + allRules := t.transfer.GetRules() log := log.WithField("row", i).WithField("role", role) + log.Trace("Transfer rules data") + + if i >= len(allRules) { + log.Warning("Invalid index") + return core.NewQVariant() + } if t.transfer == nil { log.Warning("Requested transfer rules data before transfer is connected") return qtcommon.NewQVariantString("") } - rule := t.transfer.GetRules()[i] + rule := allRules[i] switch role { case MboxIsActive: @@ -160,6 +163,9 @@ func (t *TransferRules) setTransfer(transfer *transfer.Transfer) { t.transfer = transfer + t.targetFoldersCache = make(map[string]*MboxList) + t.targetLabelsCache = make(map[string]*MboxList) + t.updateGroupSelection() } @@ -196,7 +202,9 @@ func (t *TransferRules) targetLabels(sourceID string) *MboxList { // Setters func (t *TransferRules) setIsGroupActive(groupName string, isActive bool) { - wantExclusive := (groupName == FolderTypeLabel) + log.WithField("group", groupName).WithField("active", isActive).Trace("Setting group as active/inactive") + + wantExclusive := (groupName == FolderTypeFolder) for _, rule := range t.transfer.GetRules() { if rule.SourceMailbox.IsExclusive != wantExclusive { continue @@ -265,6 +273,7 @@ func (t *TransferRules) addTargetID(sourceID string, targetID string) { newTargetMailboxes = append(newTargetMailboxes, *targetMailboxToAdd) } t.setRule(rule.SourceMailbox, newTargetMailboxes, rule.FromTime, rule.ToTime, []int{RuleTargetLabelColors}) + t.updateTargetSelection(sourceID, targetMailboxToAdd.IsExclusive) } func (t *TransferRules) removeTargetID(sourceID string, targetID string) { @@ -286,10 +295,14 @@ func (t *TransferRules) removeTargetID(sourceID string, targetID string) { } } t.setRule(rule.SourceMailbox, newTargetMailboxes, rule.FromTime, rule.ToTime, []int{RuleTargetLabelColors}) + t.updateTargetSelection(sourceID, targetMailboxToRemove.IsExclusive) } // Helpers +// getRule returns rule for given source ID. +// WARN: Always get new rule after change because previous pointer points to +// outdated struct with old data. func (t *TransferRules) getRule(sourceID string) *transfer.Rule { mailbox := t.getMailbox(t.transfer.SourceMailboxes, sourceID) if mailbox == nil { @@ -331,20 +344,19 @@ func (t *TransferRules) unsetRule(sourceMailbox transfer.Mailbox) { } func (t *TransferRules) ruleChanged(sourceMailbox transfer.Mailbox, changedRoles []int) { - for row, rule := range t.transfer.GetRules() { + allRules := t.transfer.GetRules() + for row, rule := range allRules { if rule.SourceMailbox.Hash() != sourceMailbox.Hash() { continue } - t.targetFolders(sourceMailbox.Hash()).itemsChanged(rule) - t.targetLabels(sourceMailbox.Hash()).itemsChanged(rule) - index := t.Index(row, 0, core.NewQModelIndex()) - if !index.IsValid() || row >= t.rowCount(index) { + if !index.IsValid() || row >= len(allRules) { log.WithField("row", row).Warning("Invalid index") return } + log.WithField("row", row).Trace("Transfer rule changed") t.DataChanged(index, index, changedRoles) break } @@ -375,3 +387,16 @@ func (t *TransferRules) updateGroupSelection() { t.SetIsLabelGroupSelected(areAllLabelsSelected) t.SetIsFolderGroupSelected(areAllFoldersSelected) } + +func (t *TransferRules) updateTargetSelection(sourceID string, updateFolderSelect bool) { + rule := t.getRule(sourceID) + if rule == nil { + return + } + + if updateFolderSelect { + t.targetFolders(rule.SourceMailbox.Hash()).itemsChanged(rule) + } else { + t.targetLabels(rule.SourceMailbox.Hash()).itemsChanged(rule) + } +} diff --git a/internal/frontend/qt-ie/ui.go b/internal/frontend/qt-ie/ui.go index c9414de8..fe2ee8bf 100644 --- a/internal/frontend/qt-ie/ui.go +++ b/internal/frontend/qt-ie/ui.go @@ -22,7 +22,6 @@ package qtie import ( "runtime" - qtcommon "github.com/ProtonMail/proton-bridge/internal/frontend/qt-common" "github.com/therecipe/qt/core" ) @@ -181,7 +180,7 @@ func (s *GoQMLInterface) SetFrontend(f *FrontendQt) { s.ConnectStartExport(f.StartExport) s.ConnectStartImport(f.StartImport) - s.ConnectCheckPathStatus(qtcommon.CheckPathStatus) + s.ConnectCheckPathStatus(CheckPathStatus) s.ConnectStartUpdate(f.StartUpdate) diff --git a/internal/frontend/qt/frontend.go b/internal/frontend/qt/frontend.go index 2748021e..0093d10e 100644 --- a/internal/frontend/qt/frontend.go +++ b/internal/frontend/qt/frontend.go @@ -43,15 +43,13 @@ import ( "github.com/ProtonMail/proton-bridge/internal/frontend/qt-common" "github.com/ProtonMail/proton-bridge/internal/frontend/types" "github.com/ProtonMail/proton-bridge/internal/preferences" + "github.com/ProtonMail/proton-bridge/internal/updates" "github.com/ProtonMail/proton-bridge/pkg/config" "github.com/ProtonMail/proton-bridge/pkg/ports" "github.com/ProtonMail/proton-bridge/pkg/useragent" - "github.com/sirupsen/logrus" - - //"github.com/ProtonMail/proton-bridge/pkg/keychain" "github.com/ProtonMail/proton-bridge/pkg/listener" "github.com/ProtonMail/proton-bridge/pkg/pmapi" - "github.com/ProtonMail/proton-bridge/pkg/updates" + "github.com/sirupsen/logrus" "github.com/kardianos/osext" "github.com/skratchdot/open-golang/open" "github.com/therecipe/qt/core" diff --git a/internal/frontend/types/types.go b/internal/frontend/types/types.go index 83d61b66..38032e9a 100644 --- a/internal/frontend/types/types.go +++ b/internal/frontend/types/types.go @@ -22,8 +22,8 @@ import ( "github.com/ProtonMail/proton-bridge/internal/bridge" "github.com/ProtonMail/proton-bridge/internal/importexport" "github.com/ProtonMail/proton-bridge/internal/transfer" + "github.com/ProtonMail/proton-bridge/internal/updates" "github.com/ProtonMail/proton-bridge/pkg/pmapi" - "github.com/ProtonMail/proton-bridge/pkg/updates" ) // PanicHandler is an interface of a type that can be used to gracefully handle panics which occur. @@ -104,7 +104,7 @@ func (b *bridgeWrap) GetUser(query string) (User, error) { return b.Bridge.GetUser(query) } -// ImportExporter is an interface of import/export needed by frontend. +// ImportExporter is an interface of import-export needed by frontend. type ImportExporter interface { UserManager @@ -121,9 +121,9 @@ type importExportWrap struct { *importexport.ImportExport } -// NewImportExportWrap wraps import/export struct into local importExportWrap +// NewImportExportWrap wraps import-export struct into local importExportWrap // to implement local interface. -// The problem is that Import/Export returns the importexport package's User +// The problem is that Import-Export returns the importexport package's User // type. Every method which returns User therefore has to be overridden to // fulfill the interface. func NewImportExportWrap(ie *importexport.ImportExport) *importExportWrap { //nolint[golint] diff --git a/internal/importexport/importexport.go b/internal/importexport/importexport.go index 0825fea9..bc813984 100644 --- a/internal/importexport/importexport.go +++ b/internal/importexport/importexport.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . -// Package importexport provides core functionality of Import/Export app. +// Package importexport provides core functionality of Import-Export app. package importexport import ( @@ -90,7 +90,7 @@ func (ie *ImportExport) ReportFile(osType, osVersion, accountName, address strin defer c.Logout() title := "[Import-Export] report file" - description := "An import/export report from the user swam down the river." + description := "An Import-Export report from the user swam down the river." report := pmapi.ReportReq{ OS: osType, @@ -120,7 +120,7 @@ func (ie *ImportExport) GetLocalImporter(address, path string) (*transfer.Transf if err != nil { return nil, err } - return transfer.New(ie.panicHandler, newImportMetricsManager(ie), ie.config.GetTransferDir(), source, target) + return transfer.New(ie.panicHandler, newImportMetricsManager(ie), ie.config.GetLogDir(), ie.config.GetTransferDir(), source, target) } // GetRemoteImporter returns transferrer from remote IMAP to ProtonMail account. @@ -133,7 +133,7 @@ func (ie *ImportExport) GetRemoteImporter(address, username, password, host, por if err != nil { return nil, err } - return transfer.New(ie.panicHandler, newImportMetricsManager(ie), ie.config.GetTransferDir(), source, target) + return transfer.New(ie.panicHandler, newImportMetricsManager(ie), ie.config.GetLogDir(), ie.config.GetTransferDir(), source, target) } // GetEMLExporter returns transferrer from ProtonMail account to local EML structure. @@ -143,7 +143,7 @@ func (ie *ImportExport) GetEMLExporter(address, path string) (*transfer.Transfer return nil, err } target := transfer.NewEMLProvider(path) - return transfer.New(ie.panicHandler, newExportMetricsManager(ie), ie.config.GetTransferDir(), source, target) + return transfer.New(ie.panicHandler, newExportMetricsManager(ie), ie.config.GetLogDir(), ie.config.GetTransferDir(), source, target) } // GetMBOXExporter returns transferrer from ProtonMail account to local MBOX structure. @@ -153,7 +153,7 @@ func (ie *ImportExport) GetMBOXExporter(address, path string) (*transfer.Transfe return nil, err } target := transfer.NewMBOXProvider(path) - return transfer.New(ie.panicHandler, newExportMetricsManager(ie), ie.config.GetTransferDir(), source, target) + return transfer.New(ie.panicHandler, newExportMetricsManager(ie), ie.config.GetLogDir(), ie.config.GetTransferDir(), source, target) } func (ie *ImportExport) getPMAPIProvider(address string) (*transfer.PMAPIProvider, error) { diff --git a/internal/importexport/store_factory.go b/internal/importexport/store_factory.go index 9ce40b93..99f1299d 100644 --- a/internal/importexport/store_factory.go +++ b/internal/importexport/store_factory.go @@ -21,7 +21,7 @@ import ( "github.com/ProtonMail/proton-bridge/internal/store" ) -// storeFactory implements dummy factory creating no store (not needed by Import/Export). +// storeFactory implements dummy factory creating no store (not needed by Import-Export). type storeFactory struct{} // New does nothing. diff --git a/internal/importexport/types.go b/internal/importexport/types.go index da00b1a5..c1c580db 100644 --- a/internal/importexport/types.go +++ b/internal/importexport/types.go @@ -22,5 +22,6 @@ import "github.com/ProtonMail/proton-bridge/internal/users" type Configer interface { users.Configer + GetLogDir() string GetTransferDir() string } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index a14b1154..0296472a 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -67,7 +67,7 @@ const ( Daily = Action("daily") ) -// Metrics related to import/export (transfer) process. +// Metrics related to import-export (transfer) process. const ( // Import is used to group import metrics. Import = Category("import") diff --git a/internal/transfer/mailbox.go b/internal/transfer/mailbox.go index db300a54..db6f2eac 100644 --- a/internal/transfer/mailbox.go +++ b/internal/transfer/mailbox.go @@ -25,6 +25,25 @@ import ( "github.com/ProtonMail/proton-bridge/pkg/pmapi" ) +var systemFolderMapping = map[string]string{ //nolint[gochecknoglobals] + "bin": "Trash", + "junk": "Spam", + "all": "All Mail", + "sent mail": "Sent", + "draft": "Drafts", + "important": "Starred", + // Add more translations. +} + +// LeastUsedColor is intended to return color for creating a new inbox or label +func LeastUsedColor(mailboxes []Mailbox) string { + usedColors := []string{} + for _, m := range mailboxes { + usedColors = append(usedColors, m.Color) + } + return pmapi.LeastUsedColor(usedColors) +} + // Mailbox is universal data holder of mailbox details for every provider. type Mailbox struct { ID string @@ -43,28 +62,10 @@ func (m Mailbox) Hash() string { return fmt.Sprintf("%x", sha256.Sum256([]byte(m.Name))) } -// LeastUsedColor is intended to return color for creating a new inbox or label -func LeastUsedColor(mailboxes []Mailbox) string { - usedColors := []string{} - for _, m := range mailboxes { - usedColors = append(usedColors, m.Color) - } - return pmapi.LeastUsedColor(usedColors) -} - // findMatchingMailboxes returns all matching mailboxes from `mailboxes`. -// Only one exclusive mailbox is returned. +// Only one exclusive mailbox is included. func (m Mailbox) findMatchingMailboxes(mailboxes []Mailbox) []Mailbox { - nameVariants := []string{} - if strings.Contains(m.Name, "/") || strings.Contains(m.Name, "|") { - for _, slashPart := range strings.Split(m.Name, "/") { - for _, part := range strings.Split(slashPart, "|") { - nameVariants = append(nameVariants, strings.ToLower(part)) - } - } - } - nameVariants = append(nameVariants, strings.ToLower(m.Name)) - + nameVariants := m.nameVariants() isExclusiveIncluded := false matches := []Mailbox{} for i := range nameVariants { @@ -83,3 +84,27 @@ func (m Mailbox) findMatchingMailboxes(mailboxes []Mailbox) []Mailbox { } return matches } + +// nameVariants returns all possible variants of the mailbox name. +// The best match (original name) is at the end of the slice. +// Variants are all in lower case. Examples: +// * Foo/bar -> [foo, bar, foo/bar] +// * x/Bin -> [x, trash, bin, x/bin] +// * a|b/c -> [a, b, c, a|b/c] +func (m Mailbox) nameVariants() (nameVariants []string) { + name := strings.ToLower(m.Name) + if strings.Contains(name, "/") || strings.Contains(name, "|") { + for _, slashPart := range strings.Split(name, "/") { + for _, part := range strings.Split(slashPart, "|") { + if mappedPart, ok := systemFolderMapping[part]; ok { + nameVariants = append(nameVariants, strings.ToLower(mappedPart)) + } + nameVariants = append(nameVariants, part) + } + } + } + if mappedName, ok := systemFolderMapping[name]; ok { + nameVariants = append(nameVariants, strings.ToLower(mappedName)) + } + return append(nameVariants, name) +} diff --git a/internal/transfer/mailbox_test.go b/internal/transfer/mailbox_test.go index 5865cfdc..ee1a8333 100644 --- a/internal/transfer/mailbox_test.go +++ b/internal/transfer/mailbox_test.go @@ -66,6 +66,7 @@ func TestLeastUsedColor(t *testing.T) { } r.Equal(t, "#7569d1", LeastUsedColor(mailboxes)) } + func TestFindMatchingMailboxes(t *testing.T) { mailboxes := []Mailbox{ {Name: "Inbox", IsExclusive: true}, @@ -75,6 +76,8 @@ func TestFindMatchingMailboxes(t *testing.T) { {Name: "hello/world", IsExclusive: true}, {Name: "Hello", IsExclusive: false}, {Name: "WORLD", IsExclusive: true}, + {Name: "Trash", IsExclusive: true}, + {Name: "Drafts", IsExclusive: true}, } tests := []struct { @@ -88,6 +91,10 @@ func TestFindMatchingMailboxes(t *testing.T) { {"hello/world", []string{"hello/world", "Hello"}}, {"hello|world", []string{"WORLD", "Hello"}}, {"nomailbox", []string{}}, + {"bin", []string{"Trash"}}, + {"root/bin", []string{"Trash"}}, + {"draft", []string{"Drafts"}}, + {"root/draft", []string{"Drafts"}}, } for _, tc := range tests { tc := tc diff --git a/internal/transfer/progress.go b/internal/transfer/progress.go index 0798bed5..b4873077 100644 --- a/internal/transfer/progress.go +++ b/internal/transfer/progress.go @@ -30,11 +30,12 @@ import ( // Import and export update progress about processing messages and progress // informs user interface, vice versa action (such as pause or resume) from // user interface is passed down to import and export. -type Progress struct { +type Progress struct { //nolint[maligned] log *logrus.Entry - lock sync.RWMutex + lock sync.Locker updateCh chan struct{} + messageCounted bool messageCounts map[string]uint messageStatuses map[string]*MessageStatus pauseReason string @@ -45,7 +46,8 @@ type Progress struct { func newProgress(log *logrus.Entry, fileReport *fileReport) Progress { return Progress{ - log: log, + log: log, + lock: &sync.Mutex{}, updateCh: make(chan struct{}), messageCounts: map[string]uint{}, @@ -57,11 +59,7 @@ func newProgress(log *logrus.Entry, fileReport *fileReport) Progress { // update is helper to notify listener for updates. func (p *Progress) update() { if p.updateCh == nil { - // If the progress was ended by fatal instead finish, we ignore error. - if p.fatalError != nil { - return - } - panic("update should not be called after finish was called") + return } // In case no one listens for an update, do not block the progress. @@ -71,17 +69,12 @@ func (p *Progress) update() { } } -// start should be called before anything starts. -func (p *Progress) start() { - p.lock.Lock() - defer p.lock.Unlock() -} - // finish should be called as the last call once everything is done. func (p *Progress) finish() { p.lock.Lock() defer p.lock.Unlock() + log.Debug("Progress finished") p.cleanUpdateCh() } @@ -90,6 +83,7 @@ func (p *Progress) fatal(err error) { p.lock.Lock() defer p.lock.Unlock() + log.WithError(err).Error("Progress finished") p.isStopped = true p.fatalError = err p.cleanUpdateCh() @@ -97,21 +91,26 @@ func (p *Progress) fatal(err error) { func (p *Progress) cleanUpdateCh() { if p.updateCh == nil { - // If the progress was ended by fatal instead finish, we ignore error. - if p.fatalError != nil { - return - } - panic("update should not be called after finish was called") + return } close(p.updateCh) p.updateCh = nil } +func (p *Progress) countsFinal() { + p.lock.Lock() + defer p.lock.Unlock() + defer p.update() + + log.Info("Estimating count finished") + p.messageCounted = true +} + func (p *Progress) updateCount(mailbox string, count uint) { p.lock.Lock() - defer p.update() defer p.lock.Unlock() + defer p.update() log.WithField("mailbox", mailbox).WithField("count", count).Debug("Mailbox count updated") p.messageCounts[mailbox] = count @@ -120,8 +119,8 @@ func (p *Progress) updateCount(mailbox string, count uint) { // addMessage should be called as soon as there is ID of the message. func (p *Progress) addMessage(messageID string, rule *Rule) { p.lock.Lock() - defer p.update() defer p.lock.Unlock() + defer p.update() p.log.WithField("id", messageID).Trace("Message added") p.messageStatuses[messageID] = &MessageStatus{ @@ -134,10 +133,15 @@ func (p *Progress) addMessage(messageID string, rule *Rule) { // messageExported should be called right before message is exported. func (p *Progress) messageExported(messageID string, body []byte, err error) { p.lock.Lock() - defer p.update() defer p.lock.Unlock() + defer p.update() + + log := p.log.WithField("id", messageID) + if err != nil { + log = log.WithError(err) + } + log.Debug("Message exported") - p.log.WithField("id", messageID).WithError(err).Debug("Message exported") status := p.messageStatuses[messageID] status.exportErr = err if err == nil { @@ -148,7 +152,7 @@ func (p *Progress) messageExported(messageID string, body []byte, err error) { status.bodyHash = fmt.Sprintf("%x", sha256.Sum256(body)) if header, err := getMessageHeader(body); err != nil { - p.log.WithField("id", messageID).WithError(err).Warning("Failed to parse headers for reporting") + log.WithError(err).Warning("Failed to parse headers for reporting") } else { status.setDetailsFromHeader(header) } @@ -163,10 +167,15 @@ func (p *Progress) messageExported(messageID string, body []byte, err error) { // messageImported should be called right after message is imported. func (p *Progress) messageImported(messageID, importID string, err error) { p.lock.Lock() - defer p.update() defer p.lock.Unlock() + defer p.update() + + log := p.log.WithField("id", messageID) + if err != nil { + log = log.WithError(err) + } + log.Debug("Message imported") - p.log.WithField("id", messageID).WithError(err).Debug("Message imported") p.messageStatuses[messageID].targetID = importID p.messageStatuses[messageID].importErr = err if err == nil { @@ -187,6 +196,8 @@ func (p *Progress) logMessage(messageID string) { // callWrap calls the callback and in case of problem it pause the process. // Then it waits for user action to fix it and click on continue or abort. +// Every function doing I/O should be wrapped by this function to provide +// stopping and pausing functionality. func (p *Progress) callWrap(callback func() error) { for { if p.shouldStop() { @@ -222,8 +233,8 @@ func (p *Progress) GetUpdateChannel() chan struct{} { // Pause pauses the progress. func (p *Progress) Pause(reason string) { p.lock.Lock() - defer p.update() defer p.lock.Unlock() + defer p.update() p.log.Info("Progress paused") p.pauseReason = reason @@ -232,8 +243,8 @@ func (p *Progress) Pause(reason string) { // Resume resumes the progress. func (p *Progress) Resume() { p.lock.Lock() - defer p.update() defer p.lock.Unlock() + defer p.update() p.log.Info("Progress resumed") p.pauseReason = "" @@ -258,8 +269,8 @@ func (p *Progress) PauseReason() string { // Stop stops the process. func (p *Progress) Stop() { p.lock.Lock() - defer p.update() defer p.lock.Unlock() + defer p.update() p.log.Info("Progress stopped") p.isStopped = true @@ -304,6 +315,12 @@ func (p *Progress) GetCounts() (failed, imported, exported, added, total uint) { p.lock.Lock() defer p.lock.Unlock() + // Return counts only once total is estimated or the process already + // ended (for a case when it ended quickly to report it correctly). + if p.updateCh != nil && !p.messageCounted { + return + } + // Include lost messages in the process only when transfer is done. includeMissing := p.updateCh == nil @@ -334,10 +351,10 @@ func (p *Progress) GenerateBugReport() []byte { return bugReport.getData() } -func (p *Progress) FileReport() (path string) { - if r := p.fileReport; r != nil { - path = r.path +// FileReport returns path to generated defailed file report. +func (p *Progress) FileReport() string { + if p.fileReport == nil { + return "" } - - return + return p.fileReport.path } diff --git a/internal/transfer/progress_test.go b/internal/transfer/progress_test.go index 3ec170d2..f1098d92 100644 --- a/internal/transfer/progress_test.go +++ b/internal/transfer/progress_test.go @@ -29,8 +29,6 @@ func TestProgressUpdateCount(t *testing.T) { progress := newProgress(log, nil) drainProgressUpdateChannel(&progress) - progress.start() - progress.updateCount("inbox", 10) progress.updateCount("archive", 20) progress.updateCount("inbox", 12) @@ -48,8 +46,6 @@ func TestProgressAddingMessages(t *testing.T) { progress := newProgress(log, nil) drainProgressUpdateChannel(&progress) - progress.start() - // msg1 has no problem. progress.addMessage("msg1", nil) progress.messageExported("msg1", []byte(""), nil) @@ -92,18 +88,16 @@ func TestProgressFinish(t *testing.T) { progress := newProgress(log, nil) drainProgressUpdateChannel(&progress) - progress.start() progress.finish() r.Nil(t, progress.updateCh) - r.Panics(t, func() { progress.addMessage("msg", nil) }) + r.NotPanics(t, func() { progress.addMessage("msg", nil) }) } func TestProgressFatalError(t *testing.T) { progress := newProgress(log, nil) drainProgressUpdateChannel(&progress) - progress.start() progress.fatal(errors.New("fatal error")) r.Nil(t, progress.updateCh) diff --git a/internal/transfer/provider_eml_source.go b/internal/transfer/provider_eml_source.go index 7eb949b8..925ebe21 100644 --- a/internal/transfer/provider_eml_source.go +++ b/internal/transfer/provider_eml_source.go @@ -36,6 +36,10 @@ func (p *EMLProvider) TransferTo(rules transferRules, progress *Progress, ch cha return } + if len(filePathsPerFolder) == 0 { + return + } + // This list is not filtered by time but instead going throgh each file // twice or keeping all in memory we will tell rough estimation which // will be updated during processing each file. @@ -46,6 +50,7 @@ func (p *EMLProvider) TransferTo(rules transferRules, progress *Progress, ch cha progress.updateCount(folderName, uint(len(filePaths))) } + progress.countsFinal() for folderName, filePaths := range filePathsPerFolder { // No error guaranteed by getFilePathsPerFolder. diff --git a/internal/transfer/provider_eml_target.go b/internal/transfer/provider_eml_target.go index 58f7b9ac..64e33dca 100644 --- a/internal/transfer/provider_eml_target.go +++ b/internal/transfer/provider_eml_target.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "os" "path/filepath" - "strings" "github.com/hashicorp/go-multierror" ) @@ -73,7 +72,7 @@ func (p *EMLProvider) createFolders(rules transferRules) error { func (p *EMLProvider) writeFile(msg Message) error { fileName := filepath.Base(msg.ID) - if !strings.HasSuffix(fileName, ".eml") { + if filepath.Ext(fileName) != ".eml" { fileName += ".eml" } diff --git a/internal/transfer/provider_imap.go b/internal/transfer/provider_imap.go index 16e7d493..0d96c04e 100644 --- a/internal/transfer/provider_imap.go +++ b/internal/transfer/provider_imap.go @@ -58,7 +58,7 @@ func (p *IMAPProvider) ID() string { // Mailboxes returns all available folder names from root of EML files. // In case the same folder name is used more than once (for example root/a/foo // and root/b/foo), it's treated as the same folder. -func (p *IMAPProvider) Mailboxes(includEmpty, includeAllMail bool) ([]Mailbox, error) { +func (p *IMAPProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) { mailboxesInfo, err := p.list() if err != nil { return nil, err @@ -73,11 +73,11 @@ func (p *IMAPProvider) Mailboxes(includEmpty, includeAllMail bool) ([]Mailbox, e break } } - if hasNoSelect || mailbox.Name == "[Gmail]" { + if hasNoSelect { continue } - if !includEmpty || true { + if !includeEmpty || true { mailboxStatus, err := p.selectIn(mailbox.Name) if err != nil { return nil, err diff --git a/internal/transfer/provider_imap_source.go b/internal/transfer/provider_imap_source.go index 82d13f1c..1894487e 100644 --- a/internal/transfer/provider_imap_source.go +++ b/internal/transfer/provider_imap_source.go @@ -72,6 +72,7 @@ func (p *IMAPProvider) loadMessageInfoMap(rules transferRules, progress *Progres res[rule.SourceMailbox.Name] = messagesInfo progress.updateCount(rule.SourceMailbox.Name, uint(len(messagesInfo))) } + progress.countsFinal() return res } @@ -109,7 +110,9 @@ func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValid return } } - id := fmt.Sprintf("%s_%d:%d", rule.SourceMailbox.Name, uidValidity, imapMessage.Uid) + id := getUniqueMessageID(rule.SourceMailbox.Name, uidValidity, imapMessage.Uid) + // We use ID as key to ensure we have every unique message only once. + // Some IMAP servers responded twice the same message... messagesInfo[id] = imapMessageInfo{ id: id, uid: imapMessage.Uid, @@ -173,6 +176,10 @@ func (p *IMAPProvider) exportMessages(rule *Rule, progress *Progress, ch chan<- items := []imap.FetchItem{imap.FetchUid, imap.FetchFlags, section.FetchItem()} processMessageCallback := func(imapMessage *imap.Message) { + if progress.shouldStop() { + return + } + id, ok := uidToID[imapMessage.Uid] // Sometimes, server sends not requested messages. @@ -217,3 +224,7 @@ func (p *IMAPProvider) exportMessage(rule *Rule, id string, imapMessage *imap.Me Targets: rule.TargetMailboxes, } } + +func getUniqueMessageID(mailboxName string, uidValidity, uid uint32) string { + return fmt.Sprintf("%s_%d:%d", mailboxName, uidValidity, uid) +} diff --git a/internal/transfer/provider_mbox_source.go b/internal/transfer/provider_mbox_source.go index 68491893..6b26e4b4 100644 --- a/internal/transfer/provider_mbox_source.go +++ b/internal/transfer/provider_mbox_source.go @@ -40,6 +40,10 @@ func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch ch return } + if len(filePathsPerFolder) == 0 { + return + } + for folderName, filePaths := range filePathsPerFolder { // No error guaranteed by getFilePathsPerFolder. rule, _ := rules.getRuleBySourceMailboxName(folderName) @@ -50,6 +54,7 @@ func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch ch p.updateCount(rule, progress, filePath) } } + progress.countsFinal() for folderName, filePaths := range filePathsPerFolder { // No error guaranteed by getFilePathsPerFolder. diff --git a/internal/transfer/provider_pmapi_source.go b/internal/transfer/provider_pmapi_source.go index 9a9c3e6f..cf6541fc 100644 --- a/internal/transfer/provider_pmapi_source.go +++ b/internal/transfer/provider_pmapi_source.go @@ -24,6 +24,7 @@ import ( pkgMessage "github.com/ProtonMail/proton-bridge/pkg/message" "github.com/ProtonMail/proton-bridge/pkg/pmapi" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) const pmapiListPageSize = 150 @@ -59,10 +60,11 @@ func (p *PMAPIProvider) loadCounts(rules transferRules, progress *Progress) { rule := rule progress.callWrap(func() error { _, total, err := p.listMessages(&pmapi.MessagesFilter{ - LabelID: rule.SourceMailbox.ID, - Begin: rule.FromTime, - End: rule.ToTime, - Limit: 0, + AddressID: p.addressID, + LabelID: rule.SourceMailbox.ID, + Begin: rule.FromTime, + End: rule.ToTime, + Limit: 0, }) if err != nil { log.WithError(err).Warning("Problem to load counts") @@ -72,10 +74,11 @@ func (p *PMAPIProvider) loadCounts(rules transferRules, progress *Progress) { return nil }) } + progress.countsFinal() } func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, skipEncryptedMessages bool) { - nextID := "" + page := 0 for { if progress.shouldStop() { break @@ -84,30 +87,33 @@ func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Mes isLastPage := true progress.callWrap(func() error { + // Would be better to filter by Begin and BeginID to be sure + // in case user deletes messages during the process, no message + // is skipped (paging is off then), but API does not support + // filtering by both mentioned fields at the same time. desc := false - pmapiMessages, count, err := p.listMessages(&pmapi.MessagesFilter{ + pmapiMessages, total, err := p.listMessages(&pmapi.MessagesFilter{ AddressID: p.addressID, LabelID: rule.SourceMailbox.ID, Begin: rule.FromTime, End: rule.ToTime, - BeginID: nextID, PageSize: pmapiListPageSize, - Page: 0, + Page: page, Sort: "ID", Desc: &desc, }) if err != nil { return err } - log.WithField("label", rule.SourceMailbox.ID).WithField("next", nextID).WithField("count", count).Debug("Listing messages") + log.WithFields(logrus.Fields{ + "label": rule.SourceMailbox.ID, + "page": page, + "total": total, + "count": len(pmapiMessages), + }).Debug("Listing messages") isLastPage = len(pmapiMessages) < pmapiListPageSize - // The first ID is the last one from the last page (= do not export twice the same one). - if nextID != "" { - pmapiMessages = pmapiMessages[1:] - } - for _, pmapiMessage := range pmapiMessages { if progress.shouldStop() { break @@ -122,9 +128,7 @@ func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Mes } } - if !isLastPage { - nextID = pmapiMessages[len(pmapiMessages)-1].ID - } + page++ return nil }) diff --git a/internal/transfer/provider_pmapi_target.go b/internal/transfer/provider_pmapi_target.go index 7e146825..40c7e92f 100644 --- a/internal/transfer/provider_pmapi_target.go +++ b/internal/transfer/provider_pmapi_target.go @@ -71,6 +71,11 @@ func (p *PMAPIProvider) TransferFrom(rules transferRules, progress *Progress, ch log.Info("Started transfer from channel to PMAPI") defer log.Info("Finished transfer from channel to PMAPI") + // Cache has to be cleared before each transfer to not contain + // old stuff from previous cancelled run. + p.importMsgReqMap = map[string]*pmapi.ImportMsgReq{} + p.importMsgReqSize = 0 + for msg := range ch { if progress.shouldStop() { break diff --git a/internal/transfer/rules.go b/internal/transfer/rules.go index 7cd067b2..bfbc572d 100644 --- a/internal/transfer/rules.go +++ b/internal/transfer/rules.go @@ -229,8 +229,8 @@ func (r *transferRules) getRule(sourceMailbox Mailbox) *Rule { return r.rules[h] } -// getRules returns all set rules. -func (r *transferRules) getRules() []*Rule { +// getSortedRules returns all set rules in order by `byRuleOrder`. +func (r *transferRules) getSortedRules() []*Rule { rules := []*Rule{} for _, rule := range r.rules { rules = append(rules, rule) diff --git a/internal/transfer/rules_test.go b/internal/transfer/rules_test.go index a1d93582..3f002279 100644 --- a/internal/transfer/rules_test.go +++ b/internal/transfer/rules_test.go @@ -239,7 +239,7 @@ func TestOrderRules(t *testing.T) { } gotMailboxNames := []string{} - for _, rule := range transferRules.getRules() { + for _, rule := range transferRules.getSortedRules() { gotMailboxNames = append(gotMailboxNames, rule.SourceMailbox.Name) } diff --git a/internal/transfer/transfer.go b/internal/transfer/transfer.go index 6d7ecf68..7ae8812b 100644 --- a/internal/transfer/transfer.go +++ b/internal/transfer/transfer.go @@ -34,10 +34,11 @@ type Transfer struct { panicHandler PanicHandler metrics MetricsManager id string - dir string + logDir string rules transferRules source SourceProvider target TargetProvider + rulesCache []*Rule sourceMboxCache []Mailbox targetMboxCache []Mailbox } @@ -47,14 +48,14 @@ type Transfer struct { // source := transfer.NewEMLProvider(...) // target := transfer.NewPMAPIProvider(...) // transfer.New(source, target, ...) -func New(panicHandler PanicHandler, metrics MetricsManager, transferDir string, source SourceProvider, target TargetProvider) (*Transfer, error) { +func New(panicHandler PanicHandler, metrics MetricsManager, logDir, rulesDir string, source SourceProvider, target TargetProvider) (*Transfer, error) { transferID := fmt.Sprintf("%x", sha256.Sum256([]byte(source.ID()+"-"+target.ID()))) - rules := loadRules(transferDir, transferID) + rules := loadRules(rulesDir, transferID) transfer := &Transfer{ panicHandler: panicHandler, metrics: metrics, id: transferID, - dir: transferDir, + logDir: logDir, rules: rules, source: source, target: target, @@ -108,16 +109,19 @@ func (t *Transfer) SetGlobalTimeLimit(fromTime, toTime int64) { // SetRule sets sourceMailbox for transfer. func (t *Transfer) SetRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error { + t.rulesCache = nil return t.rules.setRule(sourceMailbox, targetMailboxes, fromTime, toTime) } // UnsetRule unsets sourceMailbox from transfer. func (t *Transfer) UnsetRule(sourceMailbox Mailbox) { + t.rulesCache = nil t.rules.unsetRule(sourceMailbox) } // ResetRules unsets all rules. func (t *Transfer) ResetRules() { + t.rulesCache = nil t.rules.reset() } @@ -128,7 +132,10 @@ func (t *Transfer) GetRule(sourceMailbox Mailbox) *Rule { // GetRules returns all set transfer rules. func (t *Transfer) GetRules() []*Rule { - return t.rules.getRules() + if t.rulesCache == nil { + t.rulesCache = t.rules.getSortedRules() + } + return t.rulesCache } // SourceMailboxes returns mailboxes available at source side. @@ -171,7 +178,7 @@ func (t *Transfer) Start() *Progress { t.metrics.Start() log := log.WithField("id", t.id) - reportFile := newFileReport(t.dir, t.id) + reportFile := newFileReport(t.logDir, t.id) progress := newProgress(log, reportFile) ch := make(chan Message) @@ -179,7 +186,6 @@ func (t *Transfer) Start() *Progress { go func() { defer t.panicHandler.HandlePanic() - progress.start() t.source.TransferTo(t.rules, &progress, ch) close(ch) }() diff --git a/pkg/updates/bridge_pubkey.gpg b/internal/updates/bridge_pubkey.gpg similarity index 100% rename from pkg/updates/bridge_pubkey.gpg rename to internal/updates/bridge_pubkey.gpg diff --git a/pkg/updates/compare_versions.go b/internal/updates/compare_versions.go similarity index 100% rename from pkg/updates/compare_versions.go rename to internal/updates/compare_versions.go diff --git a/pkg/updates/compare_versions_test.go b/internal/updates/compare_versions_test.go similarity index 100% rename from pkg/updates/compare_versions_test.go rename to internal/updates/compare_versions_test.go diff --git a/pkg/updates/downloader.go b/internal/updates/downloader.go similarity index 100% rename from pkg/updates/downloader.go rename to internal/updates/downloader.go diff --git a/pkg/updates/progress.go b/internal/updates/progress.go similarity index 100% rename from pkg/updates/progress.go rename to internal/updates/progress.go diff --git a/pkg/updates/signature.go b/internal/updates/signature.go similarity index 100% rename from pkg/updates/signature.go rename to internal/updates/signature.go diff --git a/pkg/updates/sync.go b/internal/updates/sync.go similarity index 100% rename from pkg/updates/sync.go rename to internal/updates/sync.go diff --git a/pkg/updates/sync_test.go b/internal/updates/sync_test.go similarity index 100% rename from pkg/updates/sync_test.go rename to internal/updates/sync_test.go diff --git a/pkg/updates/tar.go b/internal/updates/tar.go similarity index 100% rename from pkg/updates/tar.go rename to internal/updates/tar.go diff --git a/pkg/updates/testdata/current_version_linux.json b/internal/updates/testdata/current_version_linux.json similarity index 100% rename from pkg/updates/testdata/current_version_linux.json rename to internal/updates/testdata/current_version_linux.json diff --git a/pkg/updates/testdata/current_version_linux.json.sig b/internal/updates/testdata/current_version_linux.json.sig similarity index 100% rename from pkg/updates/testdata/current_version_linux.json.sig rename to internal/updates/testdata/current_version_linux.json.sig diff --git a/pkg/updates/updates.go b/internal/updates/updates.go similarity index 98% rename from pkg/updates/updates.go rename to internal/updates/updates.go index 07463d49..e6dcfc5b 100644 --- a/pkg/updates/updates.go +++ b/internal/updates/updates.go @@ -93,7 +93,7 @@ func NewBridge(updateTempDir string) *Updates { } } -// NewImportExport inits Updates struct for import/export. +// NewImportExport inits Updates struct for import-export. func NewImportExport(updateTempDir string) *Updates { return &Updates{ version: constants.Version, @@ -102,7 +102,7 @@ func NewImportExport(updateTempDir string) *Updates { releaseNotes: importexport.ReleaseNotes, releaseFixedBugs: importexport.ReleaseFixedBugs, updateTempDir: updateTempDir, - landingPagePath: "blog/import-export-beta/", + landingPagePath: "import-export", installerFileBaseName: "Import-Export-Installer", versionFileBaseName: "current_version_ie", updateFileBaseName: "ie_upgrade", diff --git a/pkg/updates/updates_beta.go b/internal/updates/updates_beta.go similarity index 100% rename from pkg/updates/updates_beta.go rename to internal/updates/updates_beta.go diff --git a/pkg/updates/updates_qa.go b/internal/updates/updates_qa.go similarity index 100% rename from pkg/updates/updates_qa.go rename to internal/updates/updates_qa.go diff --git a/pkg/updates/updates_test.go b/internal/updates/updates_test.go similarity index 100% rename from pkg/updates/updates_test.go rename to internal/updates/updates_test.go diff --git a/pkg/updates/version_info.go b/internal/updates/version_info.go similarity index 100% rename from pkg/updates/version_info.go rename to internal/updates/version_info.go diff --git a/internal/users/credentials/credentials.go b/internal/users/credentials/credentials.go index f6736cc7..7b67b520 100644 --- a/internal/users/credentials/credentials.go +++ b/internal/users/credentials/credentials.go @@ -34,7 +34,7 @@ const ( sep = "\x00" itemLengthBridge = 9 - itemLengthImportExport = 6 // Old format for Import/Export. + itemLengthImportExport = 6 // Old format for Import-Export. ) var ( diff --git a/internal/users/users.go b/internal/users/users.go index 95ad0402..e0c18e2d 100644 --- a/internal/users/users.go +++ b/internal/users/users.go @@ -299,12 +299,11 @@ func (u *Users) addNewUser(apiUser *pmapi.User, auth *pmapi.Auth, hashedPassphra return errors.Wrap(err, "failed to update API user") } - emails := []string{} - for _, address := range client.Addresses() { - if u.useOnlyActiveAddresses && address.Receive != pmapi.CanReceive { - continue - } - emails = append(emails, address.Email) + var emails []string //nolint[prealloc] + if u.useOnlyActiveAddresses { + emails = client.Addresses().ActiveEmails() + } else { + emails = client.Addresses().AllEmails() } if _, err = u.credStorer.Add(apiUser.ID, apiUser.Name, auth.GenToken(), hashedPassphrase, emails); err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index f60f0969..5a6a0cf9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -200,7 +200,7 @@ func (c *Config) GetTLSKeyPath() string { // GetDBDir returns folder for db files. func (c *Config) GetDBDir() string { - return filepath.Join(c.appDirsVersion.UserCache()) + return c.appDirsVersion.UserCache() } // GetEventsPath returns path to events file containing the last processed event IDs. @@ -228,9 +228,9 @@ func (c *Config) GetPreferencesPath() string { return filepath.Join(c.appDirsVersion.UserCache(), "prefs.json") } -// GetTransferDir returns folder for import/export rule and report files. +// GetTransferDir returns folder for import-export rules files. func (c *Config) GetTransferDir() string { - return filepath.Join(c.appDirsVersion.UserCache()) + return c.appDirsVersion.UserCache() } // GetDefaultAPIPort returns default Bridge local API port. diff --git a/pkg/message/build.go b/pkg/message/build.go index 290da24e..f5e8422a 100644 --- a/pkg/message/build.go +++ b/pkg/message/build.go @@ -39,13 +39,13 @@ type Builder struct { cl pmapi.Client msg *pmapi.Message - EncryptedToHTML bool - succDcrpt bool + EncryptedToHTML bool + successfullyDecrypted bool } // NewBuilder initiated with client and message meta info. func NewBuilder(client pmapi.Client, message *pmapi.Message) *Builder { - return &Builder{cl: client, msg: message, EncryptedToHTML: true, succDcrpt: false} + return &Builder{cl: client, msg: message, EncryptedToHTML: true, successfullyDecrypted: false} } // fetchMessage will update original PM message if successful @@ -212,7 +212,7 @@ func (bld *Builder) BuildMessage() (structure *BodyStructure, message []byte, er } // SuccessfullyDecrypted is true when message was fetched and decrypted successfully -func (bld *Builder) SuccessfullyDecrypted() bool { return bld.succDcrpt } +func (bld *Builder) SuccessfullyDecrypted() bool { return bld.successfullyDecrypted } // WriteBody decrypts PM message and writes main body section. The external PGP // message is written as is (including attachments) @@ -225,7 +225,7 @@ func (bld *Builder) WriteBody(w io.Writer) error { if err := bld.msg.Decrypt(kr); err != nil && err != openpgperrors.ErrSignatureExpired { return err } - bld.succDcrpt = true + bld.successfullyDecrypted = true if bld.msg.MIMEType != pmapi.ContentTypeMultipartMixed { // transfer encoding qp := quotedprintable.NewWriter(w) diff --git a/pkg/pmapi/addresses.go b/pkg/pmapi/addresses.go index c4841f9f..56c47e59 100644 --- a/pkg/pmapi/addresses.go +++ b/pkg/pmapi/addresses.go @@ -95,6 +95,15 @@ func (l AddressList) ByID(id string) *Address { return nil } +// AllEmails returns all emails. +func (l AddressList) AllEmails() (addresses []string) { + for _, a := range l { + addresses = append(addresses, a.Email) + } + return +} + +// ActiveEmails returns only active emails. func (l AddressList) ActiveEmails() (addresses []string) { for _, a := range l { if a.Receive == CanReceive { diff --git a/pkg/pmapi/clientmanager.go b/pkg/pmapi/clientmanager.go index a2a91e88..765c9793 100644 --- a/pkg/pmapi/clientmanager.go +++ b/pkg/pmapi/clientmanager.go @@ -452,11 +452,10 @@ func (cm *ClientManager) HandleAuth(ca ClientAuth) { if ca.Auth == nil { cm.clearToken(ca.UserID) go cm.LogoutClient(ca.UserID) - return + } else { + cm.setToken(ca.UserID, ca.Auth.GenToken(), time.Duration(ca.Auth.ExpiresIn)*time.Second) } - cm.setToken(ca.UserID, ca.Auth.GenToken(), time.Duration(ca.Auth.ExpiresIn)*time.Second) - logrus.Debug("ClientManager is forwarding auth update...") cm.authUpdates <- ca logrus.Debug("Auth update was forwarded") diff --git a/test/context/context.go b/test/context/context.go index b614625c..59cf67ad 100644 --- a/test/context/context.go +++ b/test/context/context.go @@ -106,7 +106,7 @@ func New(app string) *TestContext { // Ensure that the config is cleaned up after the test is over. ctx.addCleanupChecked(cfg.ClearData, "Cleaning bridge config data") - // Create bridge or import/export instance under test. + // Create bridge or import-export instance under test. switch app { case "bridge": ctx.withBridgeInstance() diff --git a/test/context/importexport.go b/test/context/importexport.go index 6b468d52..ed943263 100644 --- a/test/context/importexport.go +++ b/test/context/importexport.go @@ -23,19 +23,19 @@ import ( "github.com/ProtonMail/proton-bridge/pkg/listener" ) -// GetImportExport returns import/export instance. +// GetImportExport returns import-export instance. func (ctx *TestContext) GetImportExport() *importexport.ImportExport { return ctx.importExport } -// withImportExportInstance creates a import/export instance for use in the test. +// withImportExportInstance creates a import-export instance for use in the test. // TestContext has this by default once called with env variable TEST_APP=ie. func (ctx *TestContext) withImportExportInstance() { ctx.importExport = newImportExportInstance(ctx.t, ctx.cfg, ctx.credStore, ctx.listener, ctx.clientManager) ctx.users = ctx.importExport.Users } -// newImportExportInstance creates a new import/export instance configured to use the given config/credstore. +// newImportExportInstance creates a new import-export instance configured to use the given config/credstore. func newImportExportInstance( t *bddT, cfg importexport.Configer, diff --git a/test/features/ie/transfer/import_export.feature b/test/features/ie/transfer/import_export.feature index 5cd5d29e..33cd9f36 100644 --- a/test/features/ie/transfer/import_export.feature +++ b/test/features/ie/transfer/import_export.feature @@ -1,4 +1,4 @@ -Feature: Import/Export +Feature: Import-Export app Background: Given there is connected user "user" And there is "user" with mailbox "Folders/Foo"