diff --git a/.golangci.yml b/.golangci.yml index 4dd92c9e..31d42525 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -21,6 +21,12 @@ issues: - gochecknoinits - gosec +linters-settings: + godox: + keywords: + - TODO + - FIXME + linters: # setting disable-all will make only explicitly enabled linters run disable-all: true diff --git a/Makefile b/Makefile index c7e43342..e9592cff 100644 --- a/Makefile +++ b/Makefile @@ -156,12 +156,17 @@ test: gofiles ./internal/frontend/cli/... \ ./internal/imap/... \ ./internal/metrics/... \ + ./internal/importexport/... \ ./internal/preferences/... \ ./internal/smtp/... \ ./internal/store/... \ + ./internal/transfer/... \ ./internal/users/... \ ./pkg/... +test-ie: + go test ./internal/transfer/... + bench: go test -run '^$$' -bench=. -memprofile bench_mem.pprof -cpuprofile bench_cpu.pprof ./internal/store go tool pprof -png -output bench_mem.png bench_mem.pprof @@ -172,6 +177,7 @@ coverage: test mocks: mockgen --package mocks github.com/ProtonMail/proton-bridge/internal/users Configer,PanicHandler,ClientManager,CredentialsStorer,StoreMaker > internal/users/mocks/mocks.go + mockgen --package mocks github.com/ProtonMail/proton-bridge/internal/transfer PanicHandler,ClientManager > internal/transfer/mocks/mocks.go mockgen --package mocks github.com/ProtonMail/proton-bridge/internal/store PanicHandler,ClientManager,BridgeUser > internal/store/mocks/mocks.go mockgen --package mocks github.com/ProtonMail/proton-bridge/pkg/listener Listener > internal/store/mocks/utils_mocks.go mockgen --package mocks github.com/ProtonMail/proton-bridge/pkg/pmapi Client > pkg/pmapi/mocks/mocks.go @@ -195,11 +201,15 @@ doc: .PHONY: gofiles # Following files are for the whole app so it makes sense to have them in bridge package. # (Options like cmd or internal were considered and bridge package is the best place for them.) -gofiles: ./internal/bridge/credits.go ./internal/bridge/release_notes.go +gofiles: ./internal/bridge/credits.go ./internal/bridge/release_notes.go ./internal/importexport/credits.go ./internal/importexport/release_notes.go ./internal/bridge/credits.go: ./utils/credits.sh go.mod - cd ./utils/ && ./credits.sh -./internal/bridge/release_notes.go: ./utils/release-notes.sh ./release-notes/notes.txt ./release-notes/bugs.txt - cd ./utils/ && ./release-notes.sh + cd ./utils/ && ./credits.sh bridge +./internal/bridge/release_notes.go: ./utils/release-notes.sh ./release-notes/notes-bridge.txt ./release-notes/bugs-bridge.txt + cd ./utils/ && ./release-notes.sh bridge +./internal/importexport/credits.go: ./utils/credits.sh go.mod + cd ./utils/ && ./credits.sh importexport +./internal/importexport/release_notes.go: ./utils/release-notes.sh ./release-notes/notes-importexport.txt ./release-notes/bugs-importexport.txt + cd ./utils/ && ./release-notes.sh importexport ## Run and debug @@ -219,6 +229,9 @@ run-nogui: clean-vendor gofiles run-nogui-cli: clean-vendor gofiles PROTONMAIL_ENV=dev go run ${BUILD_FLAGS_NOGUI} cmd/Desktop-Bridge/main.go ${RUN_FLAGS} -c +run-ie: + PROTONMAIL_ENV=dev go run ${BUILD_FLAGS_NOGUI} cmd/Import-Export/main.go ${RUN_FLAGS} -c + run-debug: PROTONMAIL_ENV=dev dlv debug --build-flags "${BUILD_FLAGS_NOGUI}" cmd/Desktop-Bridge/main.go -- ${RUN_FLAGS} diff --git a/cmd/Desktop-Bridge/main.go b/cmd/Desktop-Bridge/main.go index bf1bc376..7b9050df 100644 --- a/cmd/Desktop-Bridge/main.go +++ b/cmd/Desktop-Bridge/main.go @@ -147,11 +147,11 @@ func (ph *panicHandler) HandlePanic() { config.HandlePanic(ph.cfg, fmt.Sprintf("Recover: %v", r)) frontend.HandlePanic() - *ph.err = cli.NewExitError("Panic and restart", 666) + *ph.err = cli.NewExitError("Panic and restart", 255) numberOfCrashes++ log.Error("Restarting after panic") restartApp() - os.Exit(666) + os.Exit(255) } // run initializes and starts everything in a precise order. diff --git a/cmd/Import-Export/main.go b/cmd/Import-Export/main.go new file mode 100644 index 00000000..36f490f3 --- /dev/null +++ b/cmd/Import-Export/main.go @@ -0,0 +1,296 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "runtime/pprof" + "strconv" + "strings" + + "github.com/ProtonMail/proton-bridge/internal/events" + "github.com/ProtonMail/proton-bridge/internal/frontend" + "github.com/ProtonMail/proton-bridge/internal/importexport" + "github.com/ProtonMail/proton-bridge/internal/users/credentials" + "github.com/ProtonMail/proton-bridge/pkg/args" + "github.com/ProtonMail/proton-bridge/pkg/config" + "github.com/ProtonMail/proton-bridge/pkg/constants" + "github.com/ProtonMail/proton-bridge/pkg/listener" + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/ProtonMail/proton-bridge/pkg/updates" + "github.com/getsentry/raven-go" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var ( + log = logrus.WithField("pkg", "main") //nolint[gochecknoglobals] + + // How many crashes in a row. + numberOfCrashes = 0 //nolint[gochecknoglobals] + + // After how many crashes import/export gives up starting. + maxAllowedCrashes = 10 //nolint[gochecknoglobals] +) + +func main() { + constants.AppShortName = "importExport" //TODO + + if err := raven.SetDSN(constants.DSNSentry); err != nil { + log.WithError(err).Errorln("Can not setup sentry DSN") + } + raven.SetRelease(constants.Revision) + + args.FilterProcessSerialNumberFromArgs() + filterRestartNumberFromArgs() + + app := cli.NewApp() + app.Name = "Protonmail Import/Export" + app.Version = constants.BuildVersion + app.Flags = []cli.Flag{ + cli.StringFlag{ + Name: "log-level, l", + Usage: "Set the log level (one of panic, fatal, error, warn, info, debug, debug-client, debug-server)"}, + cli.BoolFlag{ + Name: "cli, c", + Usage: "Use command line interface"}, + cli.StringFlag{ + Name: "version-json, g", + Usage: "Generate json version file"}, + cli.BoolFlag{ + Name: "mem-prof, m", + Usage: "Generate memory profile"}, + cli.BoolFlag{ + Name: "cpu-prof, p", + Usage: "Generate CPU profile"}, + } + app.Usage = "ProtonMail Import/Export" + app.Action = run + + // Always log the basic info about current import/export. + logrus.SetLevel(logrus.InfoLevel) + log.WithField("version", constants.Version). + WithField("revision", constants.Revision). + WithField("runtime", runtime.GOOS). + WithField("build", constants.BuildTime). + WithField("args", os.Args). + WithField("appLong", app.Name). + WithField("appShort", constants.AppShortName). + Info("Run app") + if err := app.Run(os.Args); err != nil { + log.Error("Program exited with error: ", err) + } +} + +type panicHandler struct { + cfg *config.Config + err *error // Pointer to error of cli action. +} + +func (ph *panicHandler) HandlePanic() { + r := recover() + if r == nil { + return + } + + config.HandlePanic(ph.cfg, fmt.Sprintf("Recover: %v", r)) + frontend.HandlePanic() + + *ph.err = cli.NewExitError("Panic and restart", 255) + numberOfCrashes++ + log.Error("Restarting after panic") + restartApp() + os.Exit(255) +} + +// run initializes and starts everything in a precise order. +// +// IMPORTANT: ***Read the comments before CHANGING the order *** +func run(context *cli.Context) (contextError error) { // nolint[funlen] + // We need to have config instance to setup a logs, panic handler, etc ... + cfg := config.New(constants.AppShortName, constants.Version, constants.Revision, "") + + // We want to know about any problem. Our PanicHandler calls sentry which is + // not dependent on anything else. If that fails, it tries to create crash + // report which will not be possible if no folder can be created. That's the + // only problem we will not be notified about in any way. + panicHandler := &panicHandler{cfg, &contextError} + defer panicHandler.HandlePanic() + + // First we need config and create necessary folder; it's dependency for everything. + if err := cfg.CreateDirs(); err != nil { + log.Fatal("Cannot create necessary folders: ", err) + } + + // Setup of logs should be as soon as possible to ensure we record every wanted report in the log. + logLevel := context.GlobalString("log-level") + _, _ = config.SetupLog(cfg, logLevel) + + // Doesn't make sense to continue when Import/Export was invoked with wrong arguments. + // We should tell that to the user before we do anything else. + if context.Args().First() != "" { + _ = cli.ShowAppHelp(context) + return cli.NewExitError("Unknown argument", 4) + } + + // It's safe to get version JSON file even when other instance is running. + // (thus we put it before check of presence of other Import/Export instance). + updates := updates.New( + constants.AppShortName, + constants.Version, + constants.Revision, + constants.BuildTime, + importexport.ReleaseNotes, + importexport.ReleaseFixedBugs, + cfg.GetUpdateDir(), + ) + + if dir := context.GlobalString("version-json"); dir != "" { + generateVersionFiles(updates, dir) + return nil + } + + // In case user wants to do CPU or memory profiles... + if doCPUProfile := context.GlobalBool("cpu-prof"); doCPUProfile { + f, err := os.Create("cpu.pprof") + if err != nil { + log.Fatal("Could not create CPU profile: ", err) + } + if err := pprof.StartCPUProfile(f); err != nil { + log.Fatal("Could not start CPU profile: ", err) + } + defer pprof.StopCPUProfile() + } + + if doMemoryProfile := context.GlobalBool("mem-prof"); doMemoryProfile { + defer makeMemoryProfile() + } + + // Now we initialize all Import/Export parts. + log.Debug("Initializing import/export...") + eventListener := listener.New() + events.SetupEvents(eventListener) + + credentialsStore, credentialsError := credentials.NewStore("import-export") + if credentialsError != nil { + log.Error("Could not get credentials store: ", credentialsError) + } + + cm := pmapi.NewClientManager(cfg.GetAPIConfig()) + + // Different build types have different roundtrippers (e.g. we want to enable + // TLS fingerprint checks in production builds). GetRoundTripper has a different + // implementation depending on whether build flag pmapi_prod is used or not. + cm.SetRoundTripper(cfg.GetRoundTripper(cm, eventListener)) + + importexportInstance := importexport.New(cfg, panicHandler, eventListener, cm, credentialsStore) + + // Decide about frontend mode before initializing rest of import/export. + var frontendMode string + switch { + case context.GlobalBool("cli"): + frontendMode = "cli" + default: + frontendMode = "qt" + } + log.WithField("mode", frontendMode).Debug("Determined frontend mode to use") + + frontend := frontend.NewImportExport(constants.Version, constants.BuildVersion, frontendMode, panicHandler, cfg, eventListener, updates, importexportInstance) + + // Last part is to start everything. + log.Debug("Starting frontend...") + if err := frontend.Loop(credentialsError); err != nil { + log.Error("Frontend failed with error: ", err) + return cli.NewExitError("Frontend error", 2) + } + + if frontend.IsAppRestarting() { + restartApp() + } + + return nil +} + +// generateVersionFiles writes a JSON file with details about current build. +// Those files are used for upgrading the app. +func generateVersionFiles(updates *updates.Updates, dir string) { + log.Info("Generating version files") + for _, goos := range []string{"windows", "darwin", "linux"} { + log.Debug("Generating JSON for ", goos) + if err := updates.CreateJSONAndSign(dir, goos); err != nil { + log.Error(err) + } + } +} + +func makeMemoryProfile() { + name := "./mem.pprof" + f, err := os.Create(name) + if err != nil { + log.Error("Could not create memory profile: ", err) + } + if abs, err := filepath.Abs(name); err == nil { + name = abs + } + log.Info("Writing memory profile to ", name) + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { + log.Error("Could not write memory profile: ", err) + } + _ = f.Close() +} + +// filterRestartNumberFromArgs removes flag with a number how many restart we already did. +// See restartApp how that number is used. +func filterRestartNumberFromArgs() { + tmp := os.Args[:0] + for i, arg := range os.Args { + if !strings.HasPrefix(arg, "--restart_") { + tmp = append(tmp, arg) + continue + } + var err error + numberOfCrashes, err = strconv.Atoi(os.Args[i][10:]) + if err != nil { + numberOfCrashes = maxAllowedCrashes + } + } + os.Args = tmp +} + +// restartApp starts a new instance in background. +func restartApp() { + if numberOfCrashes >= maxAllowedCrashes { + log.Error("Too many crashes") + return + } + if exeFile, err := os.Executable(); err == nil { + arguments := append(os.Args[1:], fmt.Sprintf("--restart_%d", numberOfCrashes)) + cmd := exec.Command(exeFile, arguments...) //nolint[gosec] + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + if err := cmd.Start(); err != nil { + log.Error("Restart failed: ", err) + } + } +} diff --git a/doc/importexport.md b/doc/importexport.md new file mode 100644 index 00000000..e8449ced --- /dev/null +++ b/doc/importexport.md @@ -0,0 +1,135 @@ +# Import/Export + +## Main blocks + +This is basic overview of the main import/export blocks. + +```mermaid +graph LR + S[ProtonMail server] + U[User] + + subgraph "Import/Export app" + Users + Frontend["Qt / CLI"] + ImportExport + Transfer + + Frontend --> ImportExport + Frontend --> Transfer + ImportExport --> Users + ImportExport --> Transfer + end + + EML --> Transfer + MBOX --> Transfer + IMAP --> Transfer + S --> Transfer + + Transfer --> EML + Transfer --> MBOX + Transfer --> S + + U --> Frontend +``` + +## Code structure + +More detailed graph of main types used in Import/Export app and connection between them. + +```mermaid +graph TD + PM[ProtonMail Server] + EML[EML] + MBOX[MBOX] + IMAP[IMAP] + + subgraph "Import/Export app" + subgraph PkgUsers + subgraph PkgCredentials + CredStore[Store] + Creds[Credentials] + + CredStore --> Creds + end + + US[Users] + U[User] + + US --> U + end + + subgraph PkgFrontend + CLI + Qt + end + + subgraph PkgImportExport + IE[ImportExport] + end + + subgraph PkgTransfer + Transfer + Rules + Progress + + Provider + LocalProvider + EMLProvider + MBOXProvider + IMAPProvider + PMAPIProvider + + Mailbox + Message + + Transfer --> |source|Provider + Transfer --> |target|Provider + Transfer --> Rules + Transfer --> Progress + + Provider --> LocalProvider + Provider --> EMLProvider + Provider --> MBOXProvider + Provider --> IMAPProvider + Provider --> PMAPIProvider + + LocalProvider --> EMLProvider + LocalProvider --> MBOXProvider + + Provider --> Mailbox + Provider --> Message + + end + + subgraph PMAPI + APIM[ClientManager] + APIC[Client] + + APIM --> APIC + end + end + + CLI --> IE + CLI --> Transfer + CLI --> Progress + Qt --> IE + Qt --> Transfer + Qt --> Progress + + U --> CredStore + U --> Creds + + US --> APIM + U --> APIM + + PMAPIProvider --> APIM + EMLProvider --> EML + MBOXProvider --> MBOX + IMAPProvider --> IMAP + + IE --> US + IE --> Transfer + + APIC --> PM +``` diff --git a/doc/index.md b/doc/index.md index 84ee9e52..b6d3a543 100644 --- a/doc/index.md +++ b/doc/index.md @@ -2,8 +2,13 @@ Documentation pages in order to read for a novice: -* [Development cycle](development.md) +## Bridge + * [Bridge code](bridge.md) * [Internal Bridge database](database.md) * [Communication between Bridge, Client and Server](communication.md) * [Encryption](encryption.md) + +## Import/Export + +* [Import/Export code](importexport.md) diff --git a/go.mod b/go.mod index d0e4bfed..167b7846 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/emersion/go-imap-quota v0.0.0-20200423100218-dcfd1b7d2b41 github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075 github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26 + github.com/emersion/go-mbox v1.0.0 github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe github.com/emersion/go-vcard v0.0.0-20190105225839-8856043f13c5 // indirect diff --git a/go.sum b/go.sum index c4bd7bcc..d58e6f62 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075 h1:z8T github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075/go.mod h1:/nybxhI8kXom8Tw6BrHMl42usALvka6meORflnnYwe4= github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26 h1:FiSb8+XBQQSkcX3ubr+1tAtlRJBYaFmRZqOAweZ9Wy8= github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26/go.mod h1:+gnnZx3Mg3MnCzZrv0eZdp5puxXQUgGT/6N6L7ShKfM= +github.com/emersion/go-mbox v1.0.0 h1:HN6aKbyqmgIfK9fS/gen+NRr2wXLSxZXWfdAIAnzQPc= +github.com/emersion/go-mbox v1.0.0/go.mod h1:Yp9IVuuOYLEuMv4yjgDHvhb5mHOcYH6x92Oas3QqEZI= github.com/emersion/go-message v0.11.1 h1:0C/S4JIXDTSfXB1vpqdimAYyK4+79fgEAMQ0dSL+Kac= github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY= github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b h1:uhWtEWBHgop1rqEk2klKaxPAkVDCXexai6hSuRQ7Nvs= diff --git a/internal/bridge/bridge.go b/internal/bridge/bridge.go index 9fc63d40..3d753afd 100644 --- a/internal/bridge/bridge.go +++ b/internal/bridge/bridge.go @@ -60,7 +60,7 @@ func New( } storeFactory := newStoreFactory(config, panicHandler, clientManager, eventListener) - u := users.New(config, panicHandler, eventListener, clientManager, credStorer, storeFactory) + u := users.New(config, panicHandler, eventListener, clientManager, credStorer, storeFactory, true) b := &Bridge{ Users: u, diff --git a/internal/frontend/autoconfig/applemail.go b/internal/frontend/autoconfig/applemail.go index faa8146a..a2eb02d4 100644 --- a/internal/frontend/autoconfig/applemail.go +++ b/internal/frontend/autoconfig/applemail.go @@ -43,7 +43,7 @@ func (c *appleMail) Name() string { return "Apple Mail" } -func (c *appleMail) Configure(imapPort, smtpPort int, imapSSL, smtpSSL bool, user types.BridgeUser, addressIndex int) error { //nolint[funlen] +func (c *appleMail) Configure(imapPort, smtpPort int, imapSSL, smtpSSL bool, user types.User, addressIndex int) error { //nolint[funlen] var addresses string var displayName string diff --git a/internal/frontend/autoconfig/autoconfig.go b/internal/frontend/autoconfig/autoconfig.go index 81a08ba9..ecfd75f1 100644 --- a/internal/frontend/autoconfig/autoconfig.go +++ b/internal/frontend/autoconfig/autoconfig.go @@ -23,7 +23,7 @@ import "github.com/ProtonMail/proton-bridge/internal/frontend/types" type AutoConfig interface { Name() string - Configure(imapPort int, smtpPort int, imapSSl, smtpSSL bool, user types.BridgeUser, addressIndex int) error + Configure(imapPort int, smtpPort int, imapSSl, smtpSSL bool, user types.User, addressIndex int) error } var available []AutoConfig //nolint[gochecknoglobals] diff --git a/internal/frontend/cli-ie/account_utils.go b/internal/frontend/cli-ie/account_utils.go new file mode 100644 index 00000000..d016ce26 --- /dev/null +++ b/internal/frontend/cli-ie/account_utils.go @@ -0,0 +1,100 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package cli + +import ( + "fmt" + "strconv" + "strings" + + "github.com/ProtonMail/proton-bridge/internal/frontend/types" + "github.com/abiosoft/ishell" +) + +// completeUsernames is a helper to complete usernames as the user types. +func (f *frontendCLI) completeUsernames(args []string) (usernames []string) { + if len(args) > 1 { + return + } + arg := "" + if len(args) == 1 { + arg = args[0] + } + for _, user := range f.ie.GetUsers() { + if strings.HasPrefix(strings.ToLower(user.Username()), strings.ToLower(arg)) { + usernames = append(usernames, user.Username()) + } + } + return +} + +// noAccountWrapper is a decorator for functions which need any account to be properly functional. +func (f *frontendCLI) noAccountWrapper(callback func(*ishell.Context)) func(*ishell.Context) { + return func(c *ishell.Context) { + users := f.ie.GetUsers() + if len(users) == 0 { + f.Println("No active accounts. Please add account to continue.") + } else { + callback(c) + } + } +} + +func (f *frontendCLI) askUserByIndexOrName(c *ishell.Context) types.User { + user := f.getUserByIndexOrName("") + if user != nil { + return user + } + + numberOfAccounts := len(f.ie.GetUsers()) + indexRange := fmt.Sprintf("number between 0 and %d", numberOfAccounts-1) + if len(c.Args) == 0 { + f.Printf("Please choose %s or username.\n", indexRange) + return nil + } + arg := c.Args[0] + user = f.getUserByIndexOrName(arg) + if user == nil { + f.Printf("Wrong input '%s'. Choose %s or username.\n", bold(arg), indexRange) + return nil + } + return user +} + +func (f *frontendCLI) getUserByIndexOrName(arg string) types.User { + users := f.ie.GetUsers() + numberOfAccounts := len(users) + if numberOfAccounts == 0 { + return nil + } + if numberOfAccounts == 1 { + return users[0] + } + if index, err := strconv.Atoi(arg); err == nil { + if index < 0 || index >= numberOfAccounts { + return nil + } + return users[index] + } + for _, user := range users { + if user.Username() == arg { + return user + } + } + return nil +} diff --git a/internal/frontend/cli-ie/accounts.go b/internal/frontend/cli-ie/accounts.go new file mode 100644 index 00000000..ae3764f3 --- /dev/null +++ b/internal/frontend/cli-ie/accounts.go @@ -0,0 +1,153 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package cli + +import ( + "strings" + + "github.com/abiosoft/ishell" +) + +func (f *frontendCLI) listAccounts(c *ishell.Context) { + spacing := "%-2d: %-20s (%-15s, %-15s)\n" + f.Printf(bold(strings.Replace(spacing, "d", "s", -1)), "#", "account", "status", "address mode") + for idx, user := range f.ie.GetUsers() { + connected := "disconnected" + if user.IsConnected() { + connected = "connected" + } + mode := "split" + if user.IsCombinedAddressMode() { + mode = "combined" + } + f.Printf(spacing, idx, user.Username(), connected, mode) + } + f.Println() +} + +func (f *frontendCLI) loginAccount(c *ishell.Context) { // nolint[funlen] + f.ShowPrompt(false) + defer f.ShowPrompt(true) + + loginName := "" + if len(c.Args) > 0 { + user := f.getUserByIndexOrName(c.Args[0]) + if user != nil { + loginName = user.GetPrimaryAddress() + } + } + + if loginName == "" { + loginName = f.readStringInAttempts("Username", c.ReadLine, isNotEmpty) + if loginName == "" { + return + } + } else { + f.Println("Username:", loginName) + } + + password := f.readStringInAttempts("Password", c.ReadPassword, isNotEmpty) + if password == "" { + return + } + + f.Println("Authenticating ... ") + client, auth, err := f.ie.Login(loginName, password) + if err != nil { + f.processAPIError(err) + return + } + + if auth.HasTwoFactor() { + twoFactor := f.readStringInAttempts("Two factor code", c.ReadLine, isNotEmpty) + if twoFactor == "" { + return + } + + _, err = client.Auth2FA(twoFactor, auth) + if err != nil { + f.processAPIError(err) + return + } + } + + mailboxPassword := password + if auth.HasMailboxPassword() { + mailboxPassword = f.readStringInAttempts("Mailbox password", c.ReadPassword, isNotEmpty) + } + if mailboxPassword == "" { + return + } + + f.Println("Adding account ...") + user, err := f.ie.FinishLogin(client, auth, mailboxPassword) + if err != nil { + log.WithField("username", loginName).WithError(err).Error("Login was unsuccessful") + f.Println("Adding account was unsuccessful:", err) + return + } + + f.Printf("Account %s was added successfully.\n", bold(user.Username())) +} + +func (f *frontendCLI) logoutAccount(c *ishell.Context) { + f.ShowPrompt(false) + defer f.ShowPrompt(true) + + user := f.askUserByIndexOrName(c) + if user == nil { + return + } + if f.yesNoQuestion("Are you sure you want to logout account " + bold(user.Username())) { + if err := user.Logout(); err != nil { + f.printAndLogError("Logging out failed: ", err) + } + } +} + +func (f *frontendCLI) deleteAccount(c *ishell.Context) { + f.ShowPrompt(false) + defer f.ShowPrompt(true) + + user := f.askUserByIndexOrName(c) + if user == nil { + return + } + if f.yesNoQuestion("Are you sure you want to " + bold("remove account "+user.Username())) { + clearCache := f.yesNoQuestion("Do you want to remove cache for this account") + if err := f.ie.DeleteUser(user.ID(), clearCache); err != nil { + f.printAndLogError("Cannot delete account: ", err) + return + } + } +} + +func (f *frontendCLI) deleteAccounts(c *ishell.Context) { + f.ShowPrompt(false) + defer f.ShowPrompt(true) + + if !f.yesNoQuestion("Do you really want remove all accounts") { + return + } + for _, user := range f.ie.GetUsers() { + if err := f.ie.DeleteUser(user.ID(), false); err != nil { + f.printAndLogError("Cannot delete account ", user.Username(), ": ", err) + } + } + c.Println("Keychain cleared") +} diff --git a/internal/frontend/cli-ie/frontend.go b/internal/frontend/cli-ie/frontend.go new file mode 100644 index 00000000..7bb11586 --- /dev/null +++ b/internal/frontend/cli-ie/frontend.go @@ -0,0 +1,233 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +// Package cli provides CLI interface of the Bridge. +package cli + +import ( + "github.com/ProtonMail/proton-bridge/internal/events" + "github.com/ProtonMail/proton-bridge/internal/frontend/types" + "github.com/ProtonMail/proton-bridge/pkg/config" + "github.com/ProtonMail/proton-bridge/pkg/listener" + + "github.com/abiosoft/ishell" + "github.com/sirupsen/logrus" +) + +var ( + log = logrus.WithField("pkg", "frontend/cli-ie") //nolint[gochecknoglobals] +) + +type frontendCLI struct { + *ishell.Shell + + config *config.Config + eventListener listener.Listener + updates types.Updater + ie types.ImportExporter + + appRestart bool +} + +// New returns a new CLI frontend configured with the given options. +func New( //nolint[funlen] + panicHandler types.PanicHandler, + config *config.Config, + eventListener listener.Listener, + updates types.Updater, + ie types.ImportExporter, +) *frontendCLI { //nolint[golint] + fe := &frontendCLI{ + Shell: ishell.New(), + + config: config, + eventListener: eventListener, + updates: updates, + ie: ie, + + appRestart: false, + } + + // Clear commands. + clearCmd := &ishell.Cmd{Name: "clear", + Help: "remove stored accounts and preferences. (alias: cl)", + Aliases: []string{"cl"}, + } + clearCmd.AddCmd(&ishell.Cmd{Name: "accounts", + Help: "remove all accounts from keychain. (aliases: k, keychain)", + Aliases: []string{"a", "k", "keychain"}, + Func: fe.deleteAccounts, + }) + fe.AddCmd(clearCmd) + + // Check commands. + checkCmd := &ishell.Cmd{Name: "check", Help: "check internet connection or new version."} + checkCmd.AddCmd(&ishell.Cmd{Name: "updates", + Help: "check for Import/Export updates. (aliases: u, v, version)", + Aliases: []string{"u", "version", "v"}, + Func: fe.checkUpdates, + }) + checkCmd.AddCmd(&ishell.Cmd{Name: "internet", + Help: "check internet connection. (aliases: i, conn, connection)", + Aliases: []string{"i", "con", "connection"}, + Func: fe.checkInternetConnection, + }) + fe.AddCmd(checkCmd) + + // Print info commands. + fe.AddCmd(&ishell.Cmd{Name: "log-dir", + Help: "print path to directory with logs. (aliases: log, logs)", + Aliases: []string{"log", "logs"}, + Func: fe.printLogDir, + }) + fe.AddCmd(&ishell.Cmd{Name: "manual", + Help: "print URL with instructions. (alias: man)", + Aliases: []string{"man"}, + Func: fe.printManual, + }) + fe.AddCmd(&ishell.Cmd{Name: "release-notes", + Help: "print release notes. (aliases: notes, fixed-bugs, bugs, ver, version)", + Aliases: []string{"notes", "fixed-bugs", "bugs", "ver", "version"}, + Func: fe.printLocalReleaseNotes, + }) + fe.AddCmd(&ishell.Cmd{Name: "credits", + Help: "print used resources.", + Func: fe.printCredits, + }) + + // Account commands. + fe.AddCmd(&ishell.Cmd{Name: "list", + Help: "print the list of accounts. (aliases: l, ls)", + Func: fe.noAccountWrapper(fe.listAccounts), + Aliases: []string{"l", "ls"}, + }) + fe.AddCmd(&ishell.Cmd{Name: "login", + Help: "login procedure to add or connect account. Optionally use index or account as parameter. (aliases: a, add, con, connect)", + Func: fe.loginAccount, + Aliases: []string{"add", "a", "con", "connect"}, + Completer: fe.completeUsernames, + }) + fe.AddCmd(&ishell.Cmd{Name: "logout", + Help: "disconnect the account. Use index or account name as parameter. (aliases: d, disconnect)", + Func: fe.noAccountWrapper(fe.logoutAccount), + Aliases: []string{"d", "disconnect"}, + Completer: fe.completeUsernames, + }) + fe.AddCmd(&ishell.Cmd{Name: "delete", + Help: "remove the account from keychain. Use index or account name as parameter. (aliases: del, rm, remove)", + Func: fe.noAccountWrapper(fe.deleteAccount), + Aliases: []string{"del", "rm", "remove"}, + Completer: fe.completeUsernames, + }) + + // Import/Export commands. + importCmd := &ishell.Cmd{Name: "import", + Help: "import messages. (alias: imp)", + Aliases: []string{"imp"}, + } + importCmd.AddCmd(&ishell.Cmd{Name: "local", + Help: "import local messages. (aliases: loc)", + Func: fe.noAccountWrapper(fe.importLocalMessages), + Aliases: []string{"loc"}, + }) + importCmd.AddCmd(&ishell.Cmd{Name: "remote", + Help: "import remote messages. (aliases: rem)", + Func: fe.noAccountWrapper(fe.importRemoteMessages), + Aliases: []string{"rem"}, + }) + fe.AddCmd(importCmd) + + exportCmd := &ishell.Cmd{Name: "export", + Help: "export messages. (alias: exp)", + Aliases: []string{"exp"}, + } + exportCmd.AddCmd(&ishell.Cmd{Name: "eml", + Help: "export messages to eml files.", + Func: fe.noAccountWrapper(fe.exportMessagesToEML), + }) + exportCmd.AddCmd(&ishell.Cmd{Name: "mbox", + Help: "export messages to mbox files.", + Func: fe.noAccountWrapper(fe.exportMessagesToMBOX), + }) + fe.AddCmd(exportCmd) + + // System commands. + fe.AddCmd(&ishell.Cmd{Name: "restart", + Help: "restart the import/export.", + Func: fe.restart, + }) + + go func() { + defer panicHandler.HandlePanic() + fe.watchEvents() + }() + fe.eventListener.RetryEmit(events.TLSCertIssue) + fe.eventListener.RetryEmit(events.ErrorEvent) + return fe +} + +func (f *frontendCLI) watchEvents() { + errorCh := f.getEventChannel(events.ErrorEvent) + internetOffCh := f.getEventChannel(events.InternetOffEvent) + internetOnCh := f.getEventChannel(events.InternetOnEvent) + addressChangedLogoutCh := f.getEventChannel(events.AddressChangedLogoutEvent) + logoutCh := f.getEventChannel(events.LogoutEvent) + certIssue := f.getEventChannel(events.TLSCertIssue) + for { + select { + case errorDetails := <-errorCh: + f.Println("Import/Export failed:", errorDetails) + case <-internetOffCh: + f.notifyInternetOff() + case <-internetOnCh: + f.notifyInternetOn() + case address := <-addressChangedLogoutCh: + f.notifyLogout(address) + case userID := <-logoutCh: + user, err := f.ie.GetUser(userID) + if err != nil { + return + } + f.notifyLogout(user.Username()) + case <-certIssue: + f.notifyCertIssue() + } + } +} + +func (f *frontendCLI) getEventChannel(event string) <-chan string { + ch := make(chan string) + f.eventListener.Add(event, ch) + return ch +} + +// IsAppRestarting returns whether the app is currently set to restart. +func (f *frontendCLI) IsAppRestarting() bool { + return f.appRestart +} + +// Loop starts the frontend loop with an interactive shell. +func (f *frontendCLI) Loop(credentialsError error) error { + if credentialsError != nil { + f.notifyCredentialsError() + return credentialsError + } + + f.Print(`Welcome to ProtonMail Import/Export interactive shell`) + f.Run() + return nil +} diff --git a/internal/frontend/cli-ie/importexport.go b/internal/frontend/cli-ie/importexport.go new file mode 100644 index 00000000..05db81bf --- /dev/null +++ b/internal/frontend/cli-ie/importexport.go @@ -0,0 +1,222 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package cli + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/ProtonMail/proton-bridge/internal/frontend/types" + "github.com/ProtonMail/proton-bridge/internal/transfer" + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/abiosoft/ishell" +) + +func (f *frontendCLI) importLocalMessages(c *ishell.Context) { + f.ShowPrompt(false) + defer f.ShowPrompt(true) + + user, path := f.getUserAndPath(c, false) + if user == nil || path == "" { + return + } + + t, err := f.ie.GetLocalImporter(user.GetPrimaryAddress(), path) + f.transfer(t, err, false, true) +} + +func (f *frontendCLI) importRemoteMessages(c *ishell.Context) { + f.ShowPrompt(false) + defer f.ShowPrompt(true) + + user := f.askUserByIndexOrName(c) + if user == nil { + return + } + + username := f.readStringInAttempts("IMAP username", c.ReadLine, isNotEmpty) + if username == "" { + return + } + password := f.readStringInAttempts("IMAP password", c.ReadPassword, isNotEmpty) + if password == "" { + return + } + host := f.readStringInAttempts("IMAP host", c.ReadLine, isNotEmpty) + if host == "" { + return + } + port := f.readStringInAttempts("IMAP port", c.ReadLine, isNotEmpty) + if port == "" { + return + } + + t, err := f.ie.GetRemoteImporter(user.GetPrimaryAddress(), username, password, host, port) + f.transfer(t, err, false, true) +} + +func (f *frontendCLI) exportMessagesToEML(c *ishell.Context) { + f.ShowPrompt(false) + defer f.ShowPrompt(true) + + user, path := f.getUserAndPath(c, true) + if user == nil || path == "" { + return + } + + t, err := f.ie.GetEMLExporter(user.GetPrimaryAddress(), path) + f.transfer(t, err, true, false) +} + +func (f *frontendCLI) exportMessagesToMBOX(c *ishell.Context) { + f.ShowPrompt(false) + defer f.ShowPrompt(true) + + user, path := f.getUserAndPath(c, true) + if user == nil || path == "" { + return + } + + t, err := f.ie.GetMBOXExporter(user.GetPrimaryAddress(), path) + f.transfer(t, err, true, false) +} + +func (f *frontendCLI) getUserAndPath(c *ishell.Context, createPath bool) (types.User, string) { + user := f.askUserByIndexOrName(c) + if user == nil { + return nil, "" + } + + path := f.readStringInAttempts("Path of EML and MBOX files", c.ReadLine, isNotEmpty) + if path == "" { + return nil, "" + } + + if createPath { + _ = os.Mkdir(path, os.ModePerm) + } + + return user, path +} + +func (f *frontendCLI) transfer(t *transfer.Transfer, err error, askSkipEncrypted bool, askGlobalMailbox bool) { + if err != nil { + f.printAndLogError("Failed to init transferrer: ", err) + return + } + + if askSkipEncrypted { + skipEncryptedMessages := f.yesNoQuestion("Skip encrypted messages") + t.SetSkipEncryptedMessages(skipEncryptedMessages) + } + + if !f.setTransferRules(t) { + return + } + + if askGlobalMailbox { + if err := f.setTransferGlobalMailbox(t); err != nil { + f.printAndLogError("Failed to create global mailbox: ", err) + return + } + } + + progress := t.Start() + for range progress.GetUpdateChannel() { + f.printTransferProgress(progress) + } + f.printTransferResult(progress) +} + +func (f *frontendCLI) setTransferGlobalMailbox(t *transfer.Transfer) error { + labelName := fmt.Sprintf("Imported %s", time.Now().Format("Jan-02-2006 15:04")) + + useGlobalLabel := f.yesNoQuestion("Use global label " + labelName) + if !useGlobalLabel { + return nil + } + + globalMailbox, err := t.CreateTargetMailbox(transfer.Mailbox{ + Name: labelName, + Color: pmapi.LabelColors[0], + IsExclusive: false, + }) + if err != nil { + return err + } + + t.SetGlobalMailbox(&globalMailbox) + return nil +} + +func (f *frontendCLI) setTransferRules(t *transfer.Transfer) bool { + f.Println("Rules:") + for _, rule := range t.GetRules() { + if !rule.Active { + continue + } + targets := strings.Join(rule.TargetMailboxNames(), ", ") + if rule.HasTimeLimit() { + f.Printf(" %-30s → %s (%s - %s)\n", rule.SourceMailbox.Name, targets, rule.FromDate(), rule.ToDate()) + } else { + f.Printf(" %-30s → %s\n", rule.SourceMailbox.Name, targets) + } + } + + return f.yesNoQuestion("Proceed") +} + +func (f *frontendCLI) printTransferProgress(progress *transfer.Progress) { + failed, imported, exported, added, total := progress.GetCounts() + f.Println(fmt.Sprintf("Progress update: %d (%d / %d) / %d, failed: %d", imported, exported, added, total, failed)) + + if progress.IsPaused() { + f.Printf("Transfer is paused bacause %s", progress.PauseReason()) + if !f.yesNoQuestion("Continue (y) or stop (n)") { + progress.Stop() + } + } +} + +func (f *frontendCLI) printTransferResult(progress *transfer.Progress) { + err := progress.GetFatalError() + if err != nil { + f.Println("Transfer failed: " + err.Error()) + return + } + + statuses := progress.GetFailedMessages() + if len(statuses) == 0 { + f.Println("Transfer finished!") + return + } + + f.Println("Transfer finished with errors:") + for _, messageStatus := range statuses { + f.Printf( + " %-17s | %-30s | %-30s\n %s: %s\n", + messageStatus.Time.Format("Jan 02 2006 15:04"), + messageStatus.From, + messageStatus.Subject, + messageStatus.SourceID, + messageStatus.GetErrorMessage(), + ) + } +} diff --git a/internal/frontend/cli-ie/system.go b/internal/frontend/cli-ie/system.go new file mode 100644 index 00000000..2df6574e --- /dev/null +++ b/internal/frontend/cli-ie/system.go @@ -0,0 +1,50 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package cli + +import ( + "github.com/abiosoft/ishell" +) + +var ( + currentPort = "" //nolint[gochecknoglobals] +) + +func (f *frontendCLI) restart(c *ishell.Context) { + if f.yesNoQuestion("Are you sure you want to restart the Import/Export") { + f.Println("Restarting Import/Export...") + f.appRestart = true + f.Stop() + } +} + +func (f *frontendCLI) checkInternetConnection(c *ishell.Context) { + if f.ie.CheckConnection() == nil { + f.Println("Internet connection is available.") + } else { + f.Println("Can not contact the server, please check you internet connection.") + } +} + +func (f *frontendCLI) printLogDir(c *ishell.Context) { + f.Println("Log files are stored in\n\n ", f.config.GetLogDir()) +} + +func (f *frontendCLI) printManual(c *ishell.Context) { + f.Println("More instructions about the Import/Export can be found at\n\n https://protonmail.com/support/categories/import-export/") +} diff --git a/internal/frontend/cli-ie/updates.go b/internal/frontend/cli-ie/updates.go new file mode 100644 index 00000000..b30da468 --- /dev/null +++ b/internal/frontend/cli-ie/updates.go @@ -0,0 +1,65 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package cli + +import ( + "strings" + + "github.com/ProtonMail/proton-bridge/internal/bridge" + "github.com/ProtonMail/proton-bridge/pkg/updates" + "github.com/abiosoft/ishell" +) + +func (f *frontendCLI) checkUpdates(c *ishell.Context) { + isUpToDate, latestVersionInfo, err := f.updates.CheckIsUpToDate() + if err != nil { + f.printAndLogError("Cannot retrieve version info: ", err) + f.checkInternetConnection(c) + return + } + if isUpToDate { + f.Println("Your version is up to date.") + } else { + f.notifyNeedUpgrade() + f.Println("") + f.printReleaseNotes(latestVersionInfo) + } +} + +func (f *frontendCLI) printLocalReleaseNotes(c *ishell.Context) { + localVersion := f.updates.GetLocalVersion() + f.printReleaseNotes(localVersion) +} + +func (f *frontendCLI) printReleaseNotes(versionInfo updates.VersionInfo) { + f.Println(bold("ProtonMail Bridge "+versionInfo.Version), "\n") + if versionInfo.ReleaseNotes != "" { + f.Println(bold("Release Notes")) + f.Println(versionInfo.ReleaseNotes) + } + if versionInfo.ReleaseFixedBugs != "" { + f.Println(bold("Fixed bugs")) + f.Println(versionInfo.ReleaseFixedBugs) + } +} + +func (f *frontendCLI) printCredits(c *ishell.Context) { + for _, pkg := range strings.Split(bridge.Credits, ";") { + f.Println(pkg) + } +} diff --git a/internal/frontend/cli-ie/utils.go b/internal/frontend/cli-ie/utils.go new file mode 100644 index 00000000..f5a97000 --- /dev/null +++ b/internal/frontend/cli-ie/utils.go @@ -0,0 +1,123 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package cli + +import ( + "strings" + + pmapi "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/fatih/color" +) + +const ( + maxInputRepeat = 2 +) + +var ( + bold = color.New(color.Bold).SprintFunc() //nolint[gochecknoglobals] +) + +func isNotEmpty(val string) bool { + return val != "" +} + +func (f *frontendCLI) yesNoQuestion(question string) bool { + f.Print(question, "? yes/"+bold("no")+": ") + yes := "yes" + answer := strings.ToLower(f.ReadLine()) + for i := 0; i < len(answer); i++ { + if i >= len(yes) || answer[i] != yes[i] { + return false // Everything else is false. + } + } + return len(answer) > 0 // Empty is false. +} + +func (f *frontendCLI) readStringInAttempts(title string, readFunc func() string, isOK func(string) bool) (value string) { + f.Printf("%s: ", title) + value = readFunc() + title = strings.ToLower(string(title[0])) + title[1:] + for i := 0; !isOK(value); i++ { + if i >= maxInputRepeat { + f.Println("Too many attempts") + return "" + } + f.Printf("Please fill %s: ", title) + value = readFunc() + } + return +} + +func (f *frontendCLI) printAndLogError(args ...interface{}) { + log.Error(args...) + f.Println(args...) +} + +func (f *frontendCLI) processAPIError(err error) { + log.Warn("API error: ", err) + switch err { + case pmapi.ErrAPINotReachable: + f.notifyInternetOff() + case pmapi.ErrUpgradeApplication: + f.notifyNeedUpgrade() + default: + f.Println("Server error:", err.Error()) + } +} + +func (f *frontendCLI) notifyInternetOff() { + f.Println("Internet connection is not available.") +} + +func (f *frontendCLI) notifyInternetOn() { + f.Println("Internet connection is available again.") +} + +func (f *frontendCLI) notifyLogout(address string) { + f.Printf("Account %s is disconnected. Login to continue using this account with email client.", address) +} + +func (f *frontendCLI) notifyNeedUpgrade() { + f.Println("Please download and install the newest version of application from", f.updates.GetDownloadLink()) +} + +func (f *frontendCLI) notifyCredentialsError() { + // Print in 80-column width. + f.Println("ProtonMail Bridge is not able to detect a supported password manager") + f.Println("(pass, gnome-keyring). Please install and set up a supported password manager") + f.Println("and restart the application.") +} + +func (f *frontendCLI) notifyCertIssue() { + // Print in 80-column width. + f.Println(`Connection security error: Your network connection to Proton services may +be insecure. + +Description: +ProtonMail Bridge was not able to establish a secure connection to Proton +servers due to a TLS certificate error. This means your connection may +potentially be insecure and susceptible to monitoring by third parties. + +Recommendation: +* If you trust your network operator, you can continue to use ProtonMail + as usual. +* If you don't trust your network operator, reconnect to ProtonMail over a VPN + (such as ProtonVPN) which encrypts your Internet connection, or use + a different network to access ProtonMail. +`) +} diff --git a/internal/frontend/cli/account_utils.go b/internal/frontend/cli/account_utils.go index c2fdaec5..68bfd02d 100644 --- a/internal/frontend/cli/account_utils.go +++ b/internal/frontend/cli/account_utils.go @@ -55,7 +55,7 @@ func (f *frontendCLI) noAccountWrapper(callback func(*ishell.Context)) func(*ish } } -func (f *frontendCLI) askUserByIndexOrName(c *ishell.Context) types.BridgeUser { +func (f *frontendCLI) askUserByIndexOrName(c *ishell.Context) types.User { user := f.getUserByIndexOrName("") if user != nil { return user @@ -76,7 +76,7 @@ func (f *frontendCLI) askUserByIndexOrName(c *ishell.Context) types.BridgeUser { return user } -func (f *frontendCLI) getUserByIndexOrName(arg string) types.BridgeUser { +func (f *frontendCLI) getUserByIndexOrName(arg string) types.User { users := f.bridge.GetUsers() numberOfAccounts := len(users) if numberOfAccounts == 0 { diff --git a/internal/frontend/cli/accounts.go b/internal/frontend/cli/accounts.go index 213f11ec..a2f815a0 100644 --- a/internal/frontend/cli/accounts.go +++ b/internal/frontend/cli/accounts.go @@ -63,7 +63,7 @@ func (f *frontendCLI) showAccountInfo(c *ishell.Context) { } } -func (f *frontendCLI) showAccountAddressInfo(user types.BridgeUser, address string) { +func (f *frontendCLI) showAccountAddressInfo(user types.User, address string) { smtpSecurity := "STARTTLS" if f.preferences.GetBool(preferences.SMTPSSLKey) { smtpSecurity = "SSL" diff --git a/internal/frontend/cli/system.go b/internal/frontend/cli/system.go index aaa3cef8..a32dcc9c 100644 --- a/internal/frontend/cli/system.go +++ b/internal/frontend/cli/system.go @@ -43,7 +43,7 @@ func (f *frontendCLI) checkInternetConnection(c *ishell.Context) { if f.bridge.CheckConnection() == nil { f.Println("Internet connection is available.") } else { - f.Println("Can not contact server please check you internet connection.") + f.Println("Can not contact the server, please check you internet connection.") } } diff --git a/internal/frontend/cli/updates.go b/internal/frontend/cli/updates.go index c72b0b76..55d35e16 100644 --- a/internal/frontend/cli/updates.go +++ b/internal/frontend/cli/updates.go @@ -26,7 +26,7 @@ import ( ) func (f *frontendCLI) checkUpdates(c *ishell.Context) { - isUpToDate, latestVersionInfo, err := f.updates.CheckIsBridgeUpToDate() + isUpToDate, latestVersionInfo, err := f.updates.CheckIsUpToDate() if err != nil { f.printAndLogError("Cannot retrieve version info: ", err) f.checkInternetConnection(c) @@ -47,7 +47,7 @@ func (f *frontendCLI) printLocalReleaseNotes(c *ishell.Context) { } func (f *frontendCLI) printReleaseNotes(versionInfo updates.VersionInfo) { - f.Println(bold("ProtonMail Bridge "+versionInfo.Version), "\n") + f.Println(bold("ProtonMail Import/Export "+versionInfo.Version), "\n") if versionInfo.ReleaseNotes != "" { f.Println(bold("Release Notes")) f.Println(versionInfo.ReleaseNotes) diff --git a/internal/frontend/frontend.go b/internal/frontend/frontend.go index 6010faef..33f660de 100644 --- a/internal/frontend/frontend.go +++ b/internal/frontend/frontend.go @@ -22,8 +22,10 @@ import ( "github.com/0xAX/notificator" "github.com/ProtonMail/proton-bridge/internal/bridge" "github.com/ProtonMail/proton-bridge/internal/frontend/cli" + cliie "github.com/ProtonMail/proton-bridge/internal/frontend/cli-ie" "github.com/ProtonMail/proton-bridge/internal/frontend/qt" "github.com/ProtonMail/proton-bridge/internal/frontend/types" + "github.com/ProtonMail/proton-bridge/internal/importexport" "github.com/ProtonMail/proton-bridge/pkg/config" "github.com/ProtonMail/proton-bridge/pkg/listener" "github.com/sirupsen/logrus" @@ -86,3 +88,37 @@ func new( return qt.New(version, buildVersion, showWindowOnStart, panicHandler, config, preferences, eventListener, updates, bridge, noEncConfirmator) } } + +// NewImportExport returns initialized frontend based on `frontendType`, which can be `cli` or `qt`. +func NewImportExport( + version, + buildVersion, + frontendType string, + panicHandler types.PanicHandler, + config *config.Config, + eventListener listener.Listener, + updates types.Updater, + ie *importexport.ImportExport, +) Frontend { + ieWrap := types.NewImportExportWrap(ie) + return newImportExport(version, buildVersion, frontendType, panicHandler, config, eventListener, updates, ieWrap) +} + +func newImportExport( + version, + buildVersion, + frontendType string, + panicHandler types.PanicHandler, + config *config.Config, + eventListener listener.Listener, + updates types.Updater, + ie types.ImportExporter, +) Frontend { + switch frontendType { + case "cli": + return cliie.New(panicHandler, config, eventListener, updates, ie) + default: + return cliie.New(panicHandler, config, eventListener, updates, ie) + //return qt.New(panicHandler, config, eventListener, updates, ie) + } +} diff --git a/internal/frontend/qt/frontend.go b/internal/frontend/qt/frontend.go index f4d84d91..57b3a3c0 100644 --- a/internal/frontend/qt/frontend.go +++ b/internal/frontend/qt/frontend.go @@ -410,7 +410,7 @@ func (s *FrontendQt) isNewVersionAvailable(showMessage bool) { go func() { defer s.panicHandler.HandlePanic() defer s.Qml.ProcessFinished() - isUpToDate, latestVersionInfo, err := s.updates.CheckIsBridgeUpToDate() + isUpToDate, latestVersionInfo, err := s.updates.CheckIsUpToDate() if err != nil { log.Warn("Can not retrieve version info: ", err) s.checkInternet() diff --git a/internal/frontend/types/types.go b/internal/frontend/types/types.go index 46558bc9..ee180249 100644 --- a/internal/frontend/types/types.go +++ b/internal/frontend/types/types.go @@ -20,6 +20,8 @@ package types import ( "github.com/ProtonMail/proton-bridge/internal/bridge" + "github.com/ProtonMail/proton-bridge/internal/importexport" + "github.com/ProtonMail/proton-bridge/internal/transfer" "github.com/ProtonMail/proton-bridge/pkg/pmapi" "github.com/ProtonMail/proton-bridge/pkg/updates" ) @@ -31,7 +33,7 @@ type PanicHandler interface { // Updater is an interface for handling Bridge upgrades. type Updater interface { - CheckIsBridgeUpToDate() (isUpToDate bool, latestVersion updates.VersionInfo, err error) + CheckIsUpToDate() (isUpToDate bool, latestVersion updates.VersionInfo, err error) GetDownloadLink() string GetLocalVersion() updates.VersionInfo StartUpgrade(currentStatus chan<- updates.Progress) @@ -41,24 +43,19 @@ type NoEncConfirmator interface { ConfirmNoEncryption(string, bool) } -// Bridger is an interface of bridge needed by frontend. -type Bridger interface { - GetCurrentClient() string - SetCurrentOS(os string) +// UserManager is an interface of users needed by frontend. +type UserManager interface { Login(username, password string) (pmapi.Client, *pmapi.Auth, error) - FinishLogin(client pmapi.Client, auth *pmapi.Auth, mailboxPassword string) (BridgeUser, error) - GetUsers() []BridgeUser - GetUser(query string) (BridgeUser, error) + FinishLogin(client pmapi.Client, auth *pmapi.Auth, mailboxPassword string) (User, error) + GetUsers() []User + GetUser(query string) (User, error) DeleteUser(userID string, clearCache bool) error - ReportBug(osType, osVersion, description, accountName, address, emailClient string) error ClearData() error - AllowProxy() - DisallowProxy() CheckConnection() error } -// BridgeUser is an interface of user needed by frontend. -type BridgeUser interface { +// User is an interface of user needed by frontend. +type User interface { ID() string Username() string IsConnected() bool @@ -70,6 +67,17 @@ type BridgeUser interface { Logout() error } +// Bridger is an interface of bridge needed by frontend. +type Bridger interface { + UserManager + + GetCurrentClient() string + SetCurrentOS(os string) + ReportBug(osType, osVersion, description, accountName, address, emailClient string) error + AllowProxy() + DisallowProxy() +} + type bridgeWrap struct { *bridge.Bridge } @@ -81,17 +89,55 @@ func NewBridgeWrap(bridge *bridge.Bridge) *bridgeWrap { //nolint[golint] return &bridgeWrap{Bridge: bridge} } -func (b *bridgeWrap) FinishLogin(client pmapi.Client, auth *pmapi.Auth, mailboxPassword string) (BridgeUser, error) { +func (b *bridgeWrap) FinishLogin(client pmapi.Client, auth *pmapi.Auth, mailboxPassword string) (User, error) { return b.Bridge.FinishLogin(client, auth, mailboxPassword) } -func (b *bridgeWrap) GetUsers() (users []BridgeUser) { +func (b *bridgeWrap) GetUsers() (users []User) { for _, user := range b.Bridge.GetUsers() { users = append(users, user) } return } -func (b *bridgeWrap) GetUser(query string) (BridgeUser, error) { +func (b *bridgeWrap) GetUser(query string) (User, error) { return b.Bridge.GetUser(query) } + +// ImportExporter is an interface of import/export needed by frontend. +type ImportExporter interface { + UserManager + + GetLocalImporter(string, string) (*transfer.Transfer, error) + GetRemoteImporter(string, string, string, string, string) (*transfer.Transfer, error) + GetEMLExporter(string, string) (*transfer.Transfer, error) + GetMBOXExporter(string, string) (*transfer.Transfer, error) +} + +type importExportWrap struct { + *importexport.ImportExport +} + +// NewImportExportWrap wraps import/export struct into local importExportWrap +// to implement local interface. +// The problem is that Import/Export returns the importexport package's User +// type. Every method which returns User therefore has to be overridden to +// fulfill the interface. +func NewImportExportWrap(ie *importexport.ImportExport) *importExportWrap { //nolint[golint] + return &importExportWrap{ImportExport: ie} +} + +func (b *importExportWrap) FinishLogin(client pmapi.Client, auth *pmapi.Auth, mailboxPassword string) (User, error) { + return b.ImportExport.FinishLogin(client, auth, mailboxPassword) +} + +func (b *importExportWrap) GetUsers() (users []User) { + for _, user := range b.ImportExport.GetUsers() { + users = append(users, user) + } + return +} + +func (b *importExportWrap) GetUser(query string) (User, error) { + return b.ImportExport.GetUser(query) +} diff --git a/internal/imap/mailbox_message.go b/internal/imap/mailbox_message.go index 85e7f0d5..730f7079 100644 --- a/internal/imap/mailbox_message.go +++ b/internal/imap/mailbox_message.go @@ -19,17 +19,14 @@ package imap import ( "bytes" - "encoding/base64" "fmt" "io" - "io/ioutil" "mime/multipart" "net/mail" "net/textproto" "regexp" "sort" "strings" - "text/template" "time" "github.com/ProtonMail/gopenpgp/v2/crypto" @@ -39,7 +36,6 @@ import ( "github.com/ProtonMail/proton-bridge/pkg/parallel" "github.com/ProtonMail/proton-bridge/pkg/pmapi" "github.com/emersion/go-imap" - "github.com/emersion/go-textwrapper" "github.com/hashicorp/go-multierror" enmime "github.com/jhillyerd/enmime" "github.com/pkg/errors" @@ -185,74 +181,8 @@ func (im *imapMailbox) CreateMessage(flags []string, date time.Time, body imap.L } func (im *imapMailbox) importMessage(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (err error) { // nolint[funlen] - b := &bytes.Buffer{} - - // Overwrite content for main header for import. - // Even if message has just simple body we should upload as multipart/mixed. - // Each part has encrypted body and header reflects the original header. - mainHeader := message.GetHeader(m) - mainHeader.Set("Content-Type", "multipart/mixed; boundary="+message.GetBoundary(m)) - mainHeader.Del("Content-Disposition") - mainHeader.Del("Content-Transfer-Encoding") - if err = writeHeader(b, mainHeader); err != nil { - return - } - mw := multipart.NewWriter(b) - if err = mw.SetBoundary(message.GetBoundary(m)); err != nil { - return - } - - // Write the body part. - bodyHeader := make(textproto.MIMEHeader) - bodyHeader.Set("Content-Type", m.MIMEType+"; charset=utf-8") - bodyHeader.Set("Content-Disposition", "inline") - bodyHeader.Set("Content-Transfer-Encoding", "7bit") - - var p io.Writer - if p, err = mw.CreatePart(bodyHeader); err != nil { - return - } - // First, encrypt the message body. - if err = m.Encrypt(kr, kr); err != nil { - return err - } - if _, err := io.WriteString(p, m.Body); err != nil { - return err - } - - // Write the attachments parts. - for i := 0; i < len(m.Attachments); i++ { - att := m.Attachments[i] - r := readers[i] - h := message.GetAttachmentHeader(att) - if p, err = mw.CreatePart(h); err != nil { - return - } - // Create line wrapper writer. - ww := textwrapper.NewRFC822(p) - - // Create base64 writer. - bw := base64.NewEncoder(base64.StdEncoding, ww) - - data, err := ioutil.ReadAll(r) - if err != nil { - return err - } - - // Create encrypted writer. - pgpMessage, err := kr.Encrypt(crypto.NewPlainMessage(data), nil) - if err != nil { - return err - } - if _, err := bw.Write(pgpMessage.GetBinary()); err != nil { - return err - } - if err := bw.Close(); err != nil { - return err - } - } - - if err := mw.Close(); err != nil { + body, err := message.BuildEncrypted(m, readers, kr) + if err != nil { return err } @@ -263,7 +193,7 @@ func (im *imapMailbox) importMessage(m *pmapi.Message, readers []io.Reader, kr * } } - return im.storeMailbox.ImportMessage(m, b.Bytes(), labels) + return im.storeMailbox.ImportMessage(m, body, labels) } func (im *imapMailbox) getMessage(storeMessage storeMessageProvider, items []imap.FetchItem) (msg *imap.Message, err error) { @@ -489,55 +419,6 @@ func (im *imapMailbox) fetchMessage(m *pmapi.Message) (err error) { return } -const customMessageTemplate = ` - - - -
- Decryption error
- Decryption of this message's encrypted content failed. - 
{{.Error}}
-
- - {{if .AttachBody}} -
- 
{{.Body}}
-
- {{- end}} - - -` - -type customMessageData struct { - Error string - AttachBody bool - Body string -} - -func (im *imapMailbox) makeCustomMessage(m *pmapi.Message, decodeError error, attachBody bool) (err error) { - t := template.Must(template.New("customMessage").Parse(customMessageTemplate)) - - b := new(bytes.Buffer) - - if err = t.Execute(b, customMessageData{ - Error: decodeError.Error(), - AttachBody: attachBody, - Body: m.Body, - }); err != nil { - return - } - - m.MIMEType = pmapi.ContentTypeHTML - m.Body = b.String() - - // NOTE: we need to set header in custom message header, so we check that is non-nil. - if m.Header == nil { - m.Header = make(mail.Header) - } - - return -} - func (im *imapMailbox) writeMessageBody(w io.Writer, m *pmapi.Message) (err error) { im.log.Trace("Writing message body") @@ -555,7 +436,7 @@ func (im *imapMailbox) writeMessageBody(w io.Writer, m *pmapi.Message) (err erro err = message.WriteBody(w, kr, m) if err != nil { - if customMessageErr := im.makeCustomMessage(m, err, true); customMessageErr != nil { + if customMessageErr := message.CustomMessage(m, err, true); customMessageErr != nil { im.log.WithError(customMessageErr).Warn("Failed to make custom message") } _, _ = io.WriteString(w, m.Body) @@ -692,7 +573,7 @@ func (im *imapMailbox) buildMessage(m *pmapi.Message) (structure *message.BodySt if errDecrypt != nil && errDecrypt != openpgperrors.ErrSignatureExpired { errNoCache.add(errDecrypt) - if customMessageErr := im.makeCustomMessage(m, errDecrypt, true); customMessageErr != nil { + if customMessageErr := message.CustomMessage(m, errDecrypt, true); customMessageErr != nil { im.log.WithError(customMessageErr).Warn("Failed to make custom message") } } @@ -708,7 +589,7 @@ func (im *imapMailbox) buildMessage(m *pmapi.Message) (structure *message.BodySt return nil, nil, err } else if err != nil { errNoCache.add(err) - if customMessageErr := im.makeCustomMessage(m, err, true); customMessageErr != nil { + if customMessageErr := message.CustomMessage(m, err, true); customMessageErr != nil { im.log.WithError(customMessageErr).Warn("Failed to make custom message") } structure, msgBody, err = im.buildMessageInner(m, kr) diff --git a/internal/importexport/credits.go b/internal/importexport/credits.go new file mode 100644 index 00000000..b3a2a91d --- /dev/null +++ b/internal/importexport/credits.go @@ -0,0 +1,22 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +// Code generated by ./credits.sh at Thu Jun 4 15:54:31 CEST 2020. DO NOT EDIT. + +package importexport + +const Credits = "github.com/0xAX/notificator;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/ProtonMail/gopenpgp;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/andybalholm/cascadia;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-resty/resty/v2;github.com/golang/mock;github.com/google/go-cmp;github.com/gopherjs/gopherjs;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/jhillyerd/enmime;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/pkg/errors;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;" diff --git a/internal/importexport/importexport.go b/internal/importexport/importexport.go new file mode 100644 index 00000000..569ce733 --- /dev/null +++ b/internal/importexport/importexport.go @@ -0,0 +1,113 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +// Package importexport provides core functionality of Import/Export app. +package importexport + +import ( + "github.com/ProtonMail/proton-bridge/internal/transfer" + "github.com/ProtonMail/proton-bridge/internal/users" + + "github.com/ProtonMail/proton-bridge/pkg/listener" + logrus "github.com/sirupsen/logrus" +) + +var ( + log = logrus.WithField("pkg", "importexport") //nolint[gochecknoglobals] +) + +type ImportExport struct { + *users.Users + + config Configer + panicHandler users.PanicHandler + clientManager users.ClientManager +} + +func New( + config Configer, + panicHandler users.PanicHandler, + eventListener listener.Listener, + clientManager users.ClientManager, + credStorer users.CredentialsStorer, +) *ImportExport { + u := users.New(config, panicHandler, eventListener, clientManager, credStorer, &storeFactory{}, false) + return &ImportExport{ + Users: u, + + config: config, + panicHandler: panicHandler, + clientManager: clientManager, + } +} + +// GetLocalImporter returns transferrer from local EML or MBOX structure to ProtonMail account. +func (ie *ImportExport) GetLocalImporter(address, path string) (*transfer.Transfer, error) { + source := transfer.NewLocalProvider(path) + target, err := ie.getPMAPIProvider(address) + if err != nil { + return nil, err + } + return transfer.New(ie.panicHandler, ie.config.GetTransferDir(), source, target) +} + +// GetRemoteImporter returns transferrer from remote IMAP to ProtonMail account. +func (ie *ImportExport) GetRemoteImporter(address, username, password, host, port string) (*transfer.Transfer, error) { + source, err := transfer.NewIMAPProvider(username, password, host, port) + if err != nil { + return nil, err + } + target, err := ie.getPMAPIProvider(address) + if err != nil { + return nil, err + } + return transfer.New(ie.panicHandler, ie.config.GetTransferDir(), source, target) +} + +// GetEMLExporter returns transferrer from ProtonMail account to local EML structure. +func (ie *ImportExport) GetEMLExporter(address, path string) (*transfer.Transfer, error) { + source, err := ie.getPMAPIProvider(address) + if err != nil { + return nil, err + } + target := transfer.NewEMLProvider(path) + return transfer.New(ie.panicHandler, ie.config.GetTransferDir(), source, target) +} + +// GetMBOXExporter returns transferrer from ProtonMail account to local MBOX structure. +func (ie *ImportExport) GetMBOXExporter(address, path string) (*transfer.Transfer, error) { + source, err := ie.getPMAPIProvider(address) + if err != nil { + return nil, err + } + target := transfer.NewMBOXProvider(path) + return transfer.New(ie.panicHandler, ie.config.GetTransferDir(), source, target) +} + +func (ie *ImportExport) getPMAPIProvider(address string) (*transfer.PMAPIProvider, error) { + user, err := ie.Users.GetUser(address) + if err != nil { + return nil, err + } + + addressID, err := user.GetAddressID(address) + if err != nil { + return nil, err + } + + return transfer.NewPMAPIProvider(ie.clientManager, user.ID(), addressID) +} diff --git a/internal/importexport/release_notes.go b/internal/importexport/release_notes.go new file mode 100644 index 00000000..456f472c --- /dev/null +++ b/internal/importexport/release_notes.go @@ -0,0 +1,26 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +// Code generated by ./release-notes.sh at 'Thu Jun 4 15:54:31 CEST 2020'. DO NOT EDIT. + +package importexport + +const ReleaseNotes = ` +` + +const ReleaseFixedBugs = ` +` diff --git a/internal/importexport/store_factory.go b/internal/importexport/store_factory.go new file mode 100644 index 00000000..9ce40b93 --- /dev/null +++ b/internal/importexport/store_factory.go @@ -0,0 +1,35 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package importexport + +import ( + "github.com/ProtonMail/proton-bridge/internal/store" +) + +// storeFactory implements dummy factory creating no store (not needed by Import/Export). +type storeFactory struct{} + +// New does nothing. +func (f *storeFactory) New(user store.BridgeUser) (*store.Store, error) { + return nil, nil +} + +// Remove does nothing. +func (f *storeFactory) Remove(userID string) error { + return nil +} diff --git a/internal/importexport/types.go b/internal/importexport/types.go new file mode 100644 index 00000000..da00b1a5 --- /dev/null +++ b/internal/importexport/types.go @@ -0,0 +1,26 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package importexport + +import "github.com/ProtonMail/proton-bridge/internal/users" + +type Configer interface { + users.Configer + + GetTransferDir() string +} diff --git a/internal/transfer/mailbox.go b/internal/transfer/mailbox.go new file mode 100644 index 00000000..556c3218 --- /dev/null +++ b/internal/transfer/mailbox.go @@ -0,0 +1,69 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "crypto/sha256" + "fmt" + "strings" +) + +// Mailbox is universal data holder of mailbox details for every provider. +type Mailbox struct { + ID string + Name string + Color string + IsExclusive bool +} + +// Hash returns unique identifier to be used for matching. +func (m Mailbox) Hash() string { + return fmt.Sprintf("%x", sha256.Sum256([]byte(m.Name))) +} + +// findMatchingMailboxes returns all matching mailboxes from `mailboxes`. +// Only one exclusive mailbox is returned. +func (m Mailbox) findMatchingMailboxes(mailboxes []Mailbox) []Mailbox { + nameVariants := []string{} + if strings.Contains(m.Name, "/") || strings.Contains(m.Name, "|") { + for _, slashPart := range strings.Split(m.Name, "/") { + for _, part := range strings.Split(slashPart, "|") { + nameVariants = append(nameVariants, strings.ToLower(part)) + } + } + } + nameVariants = append(nameVariants, strings.ToLower(m.Name)) + + isExclusiveIncluded := false + matches := []Mailbox{} + for i := range nameVariants { + nameVariant := nameVariants[len(nameVariants)-1-i] + for _, mailbox := range mailboxes { + if mailbox.IsExclusive && isExclusiveIncluded { + continue + } + if strings.ToLower(mailbox.Name) == nameVariant { + matches = append(matches, mailbox) + if mailbox.IsExclusive { + isExclusiveIncluded = true + } + } + } + } + return matches +} diff --git a/internal/transfer/mailbox_test.go b/internal/transfer/mailbox_test.go new file mode 100644 index 00000000..c59b4581 --- /dev/null +++ b/internal/transfer/mailbox_test.go @@ -0,0 +1,61 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "testing" + + r "github.com/stretchr/testify/require" +) + +func TestFindMatchingMailboxes(t *testing.T) { + mailboxes := []Mailbox{ + {Name: "Inbox", IsExclusive: true}, + {Name: "Sent", IsExclusive: true}, + {Name: "Archive", IsExclusive: true}, + {Name: "Foo", IsExclusive: false}, + {Name: "hello/world", IsExclusive: true}, + {Name: "Hello", IsExclusive: false}, + {Name: "WORLD", IsExclusive: true}, + } + + tests := []struct { + name string + wantNames []string + }{ + {"inbox", []string{"Inbox"}}, + {"foo", []string{"Foo"}}, + {"hello", []string{"Hello"}}, + {"world", []string{"WORLD"}}, + {"hello/world", []string{"hello/world", "Hello"}}, + {"hello|world", []string{"WORLD", "Hello"}}, + {"nomailbox", []string{}}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + mailbox := Mailbox{Name: tc.name} + got := mailbox.findMatchingMailboxes(mailboxes) + gotNames := []string{} + for _, m := range got { + gotNames = append(gotNames, m.Name) + } + r.Equal(t, tc.wantNames, gotNames) + }) + } +} diff --git a/internal/transfer/message.go b/internal/transfer/message.go new file mode 100644 index 00000000..5c939305 --- /dev/null +++ b/internal/transfer/message.go @@ -0,0 +1,100 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "fmt" + "mime" + "net/mail" + "time" +) + +// Message is data holder passed between import and export. +type Message struct { + ID string + Unread bool + Body []byte + Source Mailbox + Targets []Mailbox +} + +// MessageStatus holds status for message used by progress manager. +type MessageStatus struct { + eventTime time.Time // Time of adding message to the process. + rule *Rule // Rule with source and target mailboxes. + SourceID string // Message ID at the source. + targetID string // Message ID at the target (if any). + bodyHash string // Hash of the message body. + + exported bool + imported bool + exportErr error + importErr error + + // Info about message displayed to user. + // This is needed only for failed messages, but we cannot know in advance + // which message will fail. We could clear it once the message passed + // without any error. However, if we say one message takes about 100 bytes + // in average, it's about 100 MB per million of messages, which is fine. + Subject string + From string + Time time.Time +} + +func (status *MessageStatus) setDetailsFromHeader(header mail.Header) { + dec := &mime.WordDecoder{} + + status.Subject = header.Get("subject") + if subject, err := dec.Decode(status.Subject); err == nil { + status.Subject = subject + } + + status.From = header.Get("from") + if from, err := dec.Decode(status.From); err == nil { + status.From = from + } + + if msgTime, err := header.Date(); err == nil { + status.Time = msgTime + } +} + +func (status *MessageStatus) hasError(includeMissing bool) bool { + return status.exportErr != nil || status.importErr != nil || (includeMissing && !status.imported) +} + +// GetErrorMessage returns error message. +func (status *MessageStatus) GetErrorMessage() string { + return status.getErrorMessage(true) +} + +func (status *MessageStatus) getErrorMessage(includeMissing bool) string { + if status.exportErr != nil { + return fmt.Sprintf("failed to export: %s", status.exportErr) + } + if status.importErr != nil { + return fmt.Sprintf("failed to import: %s", status.importErr) + } + if includeMissing && !status.imported { + if !status.exported { + return "failed to import: lost before read" + } + return "failed to import: lost in the process" + } + return "" +} diff --git a/internal/transfer/mocks/mocks.go b/internal/transfer/mocks/mocks.go new file mode 100644 index 00000000..bc18f2b7 --- /dev/null +++ b/internal/transfer/mocks/mocks.go @@ -0,0 +1,97 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/ProtonMail/proton-bridge/internal/transfer (interfaces: PanicHandler,ClientManager) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + pmapi "github.com/ProtonMail/proton-bridge/pkg/pmapi" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockPanicHandler is a mock of PanicHandler interface +type MockPanicHandler struct { + ctrl *gomock.Controller + recorder *MockPanicHandlerMockRecorder +} + +// MockPanicHandlerMockRecorder is the mock recorder for MockPanicHandler +type MockPanicHandlerMockRecorder struct { + mock *MockPanicHandler +} + +// NewMockPanicHandler creates a new mock instance +func NewMockPanicHandler(ctrl *gomock.Controller) *MockPanicHandler { + mock := &MockPanicHandler{ctrl: ctrl} + mock.recorder = &MockPanicHandlerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockPanicHandler) EXPECT() *MockPanicHandlerMockRecorder { + return m.recorder +} + +// HandlePanic mocks base method +func (m *MockPanicHandler) HandlePanic() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "HandlePanic") +} + +// HandlePanic indicates an expected call of HandlePanic +func (mr *MockPanicHandlerMockRecorder) HandlePanic() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandlePanic", reflect.TypeOf((*MockPanicHandler)(nil).HandlePanic)) +} + +// MockClientManager is a mock of ClientManager interface +type MockClientManager struct { + ctrl *gomock.Controller + recorder *MockClientManagerMockRecorder +} + +// MockClientManagerMockRecorder is the mock recorder for MockClientManager +type MockClientManagerMockRecorder struct { + mock *MockClientManager +} + +// NewMockClientManager creates a new mock instance +func NewMockClientManager(ctrl *gomock.Controller) *MockClientManager { + mock := &MockClientManager{ctrl: ctrl} + mock.recorder = &MockClientManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockClientManager) EXPECT() *MockClientManagerMockRecorder { + return m.recorder +} + +// CheckConnection mocks base method +func (m *MockClientManager) CheckConnection() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckConnection") + ret0, _ := ret[0].(error) + return ret0 +} + +// CheckConnection indicates an expected call of CheckConnection +func (mr *MockClientManagerMockRecorder) CheckConnection() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckConnection", reflect.TypeOf((*MockClientManager)(nil).CheckConnection)) +} + +// GetClient mocks base method +func (m *MockClientManager) GetClient(arg0 string) pmapi.Client { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetClient", arg0) + ret0, _ := ret[0].(pmapi.Client) + return ret0 +} + +// GetClient indicates an expected call of GetClient +func (mr *MockClientManagerMockRecorder) GetClient(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClient", reflect.TypeOf((*MockClientManager)(nil).GetClient), arg0) +} diff --git a/internal/transfer/progress.go b/internal/transfer/progress.go new file mode 100644 index 00000000..ee463f45 --- /dev/null +++ b/internal/transfer/progress.go @@ -0,0 +1,331 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "crypto/sha256" + "fmt" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +// Progress maintains progress between import, export and user interface. +// Import and export update progress about processing messages and progress +// informs user interface, vice versa action (such as pause or resume) from +// user interface is passed down to import and export. +type Progress struct { + log *logrus.Entry + lock sync.RWMutex + + updateCh chan struct{} + messageCounts map[string]uint + messageStatuses map[string]*MessageStatus + pauseReason string + isStopped bool + fatalError error + fileReport *fileReport +} + +func newProgress(log *logrus.Entry, fileReport *fileReport) Progress { + return Progress{ + log: log, + + updateCh: make(chan struct{}), + messageCounts: map[string]uint{}, + messageStatuses: map[string]*MessageStatus{}, + fileReport: fileReport, + } +} + +// update is helper to notify listener for updates. +func (p *Progress) update() { + if p.updateCh == nil { + // If the progress was ended by fatal instead finish, we ignore error. + if p.fatalError != nil { + return + } + panic("update should not be called after finish was called") + } + + // In case no one listens for an update, do not block the progress. + select { + case p.updateCh <- struct{}{}: + case <-time.After(100 * time.Millisecond): + } +} + +// start should be called before anything starts. +func (p *Progress) start() { + p.lock.Lock() + defer p.lock.Unlock() +} + +// finish should be called as the last call once everything is done. +func (p *Progress) finish() { + p.lock.Lock() + defer p.lock.Unlock() + + p.cleanUpdateCh() +} + +// fatal should be called once there is error with no possible continuation. +func (p *Progress) fatal(err error) { + p.lock.Lock() + defer p.lock.Unlock() + + p.isStopped = true + p.fatalError = err + p.cleanUpdateCh() +} + +func (p *Progress) cleanUpdateCh() { + if p.updateCh == nil { + // If the progress was ended by fatal instead finish, we ignore error. + if p.fatalError != nil { + return + } + panic("update should not be called after finish was called") + } + + close(p.updateCh) + p.updateCh = nil +} + +func (p *Progress) updateCount(mailbox string, count uint) { + p.lock.Lock() + defer p.update() + defer p.lock.Unlock() + + log.WithField("mailbox", mailbox).WithField("count", count).Debug("Mailbox count updated") + p.messageCounts[mailbox] = count +} + +// addMessage should be called as soon as there is ID of the message. +func (p *Progress) addMessage(messageID string, rule *Rule) { + p.lock.Lock() + defer p.update() + defer p.lock.Unlock() + + p.log.WithField("id", messageID).Trace("Message added") + p.messageStatuses[messageID] = &MessageStatus{ + eventTime: time.Now(), + rule: rule, + SourceID: messageID, + } +} + +// messageExported should be called right before message is exported. +func (p *Progress) messageExported(messageID string, body []byte, err error) { + p.lock.Lock() + defer p.update() + defer p.lock.Unlock() + + p.log.WithField("id", messageID).WithError(err).Debug("Message exported") + status := p.messageStatuses[messageID] + status.exported = true + status.exportErr = err + + if len(body) > 0 { + status.bodyHash = fmt.Sprintf("%x", sha256.Sum256(body)) + + if header, err := getMessageHeader(body); err != nil { + p.log.WithField("id", messageID).WithError(err).Warning("Failed to parse headers for reporting") + } else { + status.setDetailsFromHeader(header) + } + } + + // If export failed, no other step will be done with message and we can log it to the report file. + if err != nil { + p.logMessage(messageID) + } +} + +// messageImported should be called right after message is imported. +func (p *Progress) messageImported(messageID, importID string, err error) { + p.lock.Lock() + defer p.update() + defer p.lock.Unlock() + + p.log.WithField("id", messageID).WithError(err).Debug("Message imported") + p.messageStatuses[messageID].targetID = importID + p.messageStatuses[messageID].imported = true + p.messageStatuses[messageID].importErr = err + + // Import is the last step, now we can log the result to the report file. + p.logMessage(messageID) +} + +// logMessage writes message status to log file. +func (p *Progress) logMessage(messageID string) { + if p.fileReport == nil { + return + } + p.fileReport.writeMessageStatus(p.messageStatuses[messageID]) +} + +// callWrap calls the callback and in case of problem it pause the process. +// Then it waits for user action to fix it and click on continue or abort. +func (p *Progress) callWrap(callback func() error) { + for { + if p.shouldStop() { + break + } + + err := callback() + if err == nil { + break + } + + p.Pause(err.Error()) + } +} + +// shouldStop is utility for providers to automatically wait during pause +// and returned value determines whether the process shouls be fully stopped. +func (p *Progress) shouldStop() bool { + for p.IsPaused() { + time.Sleep(time.Second) + } + return p.IsStopped() +} + +// GetUpdateChannel returns channel notifying any update from import or export. +func (p *Progress) GetUpdateChannel() chan struct{} { + p.lock.Lock() + defer p.lock.Unlock() + + return p.updateCh +} + +// Pause pauses the progress. +func (p *Progress) Pause(reason string) { + p.lock.Lock() + defer p.update() + defer p.lock.Unlock() + + p.log.Info("Progress paused") + p.pauseReason = reason +} + +// Resume resumes the progress. +func (p *Progress) Resume() { + p.lock.Lock() + defer p.update() + defer p.lock.Unlock() + + p.log.Info("Progress resumed") + p.pauseReason = "" +} + +// IsPaused returns whether progress is paused. +func (p *Progress) IsPaused() bool { + p.lock.Lock() + defer p.lock.Unlock() + + return p.pauseReason != "" +} + +// PauseReason returns pause reason. +func (p *Progress) PauseReason() string { + p.lock.Lock() + defer p.lock.Unlock() + + return p.pauseReason +} + +// Stop stops the process. +func (p *Progress) Stop() { + p.lock.Lock() + defer p.update() + defer p.lock.Unlock() + + p.log.Info("Progress stopped") + p.isStopped = true + p.pauseReason = "" // Clear pause to run paused code and stop it. +} + +// IsStopped returns whether progress is stopped. +func (p *Progress) IsStopped() bool { + p.lock.Lock() + defer p.lock.Unlock() + + return p.isStopped +} + +// GetFatalError returns fatal error (progress failed and did not finish). +func (p *Progress) GetFatalError() error { + p.lock.Lock() + defer p.lock.Unlock() + + return p.fatalError +} + +// GetFailedMessages returns statuses of failed messages. +func (p *Progress) GetFailedMessages() []*MessageStatus { + p.lock.Lock() + defer p.lock.Unlock() + + // Include lost messages in the process only when transfer is done. + includeMissing := p.updateCh == nil + + statuses := []*MessageStatus{} + for _, status := range p.messageStatuses { + if status.hasError(includeMissing) { + statuses = append(statuses, status) + } + } + return statuses +} + +// GetCounts returns counts of exported and imported messages. +func (p *Progress) GetCounts() (failed, imported, exported, added, total uint) { + p.lock.Lock() + defer p.lock.Unlock() + + // Include lost messages in the process only when transfer is done. + includeMissing := p.updateCh == nil + + for _, mailboxCount := range p.messageCounts { + total += mailboxCount + } + for _, status := range p.messageStatuses { + added++ + if status.exported { + exported++ + } + if status.imported { + imported++ + } + if status.hasError(includeMissing) { + failed++ + } + } + return +} + +// GenerateBugReport generates similar file to import log except private information. +func (p *Progress) GenerateBugReport() []byte { + bugReport := bugReport{} + for _, status := range p.messageStatuses { + bugReport.writeMessageStatus(status) + } + return bugReport.getData() +} diff --git a/internal/transfer/progress_test.go b/internal/transfer/progress_test.go new file mode 100644 index 00000000..68baeb16 --- /dev/null +++ b/internal/transfer/progress_test.go @@ -0,0 +1,120 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "testing" + + "github.com/pkg/errors" + a "github.com/stretchr/testify/assert" + r "github.com/stretchr/testify/require" +) + +func TestProgressUpdateCount(t *testing.T) { + progress := newProgress(log, nil) + drainProgressUpdateChannel(&progress) + + progress.start() + + progress.updateCount("inbox", 10) + progress.updateCount("archive", 20) + progress.updateCount("inbox", 12) + progress.updateCount("sent", 5) + progress.updateCount("foo", 4) + progress.updateCount("foo", 5) + + progress.finish() + + _, _, _, _, total := progress.GetCounts() //nolint[dogsled] + r.Equal(t, uint(42), total) +} + +func TestProgressAddingMessages(t *testing.T) { + progress := newProgress(log, nil) + drainProgressUpdateChannel(&progress) + + progress.start() + + // msg1 has no problem. + progress.addMessage("msg1", nil) + progress.messageExported("msg1", []byte(""), nil) + progress.messageImported("msg1", "", nil) + + // msg2 has an import problem. + progress.addMessage("msg2", nil) + progress.messageExported("msg2", []byte(""), nil) + progress.messageImported("msg2", "", errors.New("failed import")) + + // msg3 has an export problem. + progress.addMessage("msg3", nil) + progress.messageExported("msg3", []byte(""), errors.New("failed export")) + + // msg4 has an export problem and import is also called. + progress.addMessage("msg4", nil) + progress.messageExported("msg4", []byte(""), errors.New("failed export")) + progress.messageImported("msg4", "", nil) + + progress.finish() + + failed, imported, exported, added, _ := progress.GetCounts() + a.Equal(t, uint(4), added) + a.Equal(t, uint(4), exported) + a.Equal(t, uint(3), imported) + a.Equal(t, uint(3), failed) + + errorsMap := map[string]string{} + for _, status := range progress.GetFailedMessages() { + errorsMap[status.SourceID] = status.GetErrorMessage() + } + a.Equal(t, map[string]string{ + "msg2": "failed to import: failed import", + "msg3": "failed to export: failed export", + "msg4": "failed to export: failed export", + }, errorsMap) +} + +func TestProgressFinish(t *testing.T) { + progress := newProgress(log, nil) + drainProgressUpdateChannel(&progress) + + progress.start() + progress.finish() + r.Nil(t, progress.updateCh) + + r.Panics(t, func() { progress.addMessage("msg", nil) }) +} + +func TestProgressFatalError(t *testing.T) { + progress := newProgress(log, nil) + drainProgressUpdateChannel(&progress) + + progress.start() + progress.fatal(errors.New("fatal error")) + r.Nil(t, progress.updateCh) + + r.NotPanics(t, func() { progress.addMessage("msg", nil) }) +} + +func drainProgressUpdateChannel(progress *Progress) { + // updateCh is not needed to drain under tests - timeout is implemented. + // But timeout takes time which would slow down tests. + go func() { + for range progress.updateCh { + } + }() +} diff --git a/internal/transfer/provider.go b/internal/transfer/provider.go new file mode 100644 index 00000000..34048013 --- /dev/null +++ b/internal/transfer/provider.go @@ -0,0 +1,51 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +// Provider provides interface for common operation with provider. +type Provider interface { + // ID is used for generating transfer ID by combining source and target ID. + ID() string + + // Mailboxes returns all available mailboxes. + // Provider used as source returns only non-empty maibloxes. + // Provider used as target does not return all mail maiblox. + Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) +} + +// SourceProvider provides interface of provider with support of export. +type SourceProvider interface { + Provider + + // TransferTo exports messages based on rules to channel. + TransferTo(transferRules, *Progress, chan<- Message) +} + +// TargetProvider provides interface of provider with support of import. +type TargetProvider interface { + Provider + + // DefaultMailboxes returns the default mailboxes for default rules if no other is found. + DefaultMailboxes(sourceMailbox Mailbox) (targetMailboxes []Mailbox) + + // CreateMailbox creates new mailbox to be used as target in transfer rules. + CreateMailbox(Mailbox) (Mailbox, error) + + // TransferFrom imports messages from channel. + TransferFrom(transferRules, *Progress, <-chan Message) +} diff --git a/internal/transfer/provider_eml.go b/internal/transfer/provider_eml.go new file mode 100644 index 00000000..051afe1f --- /dev/null +++ b/internal/transfer/provider_eml.go @@ -0,0 +1,65 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +// EMLProvider implements import and export to/from EML file structure. +type EMLProvider struct { + root string +} + +// NewEMLProvider creates EMLProvider. +func NewEMLProvider(root string) *EMLProvider { + return &EMLProvider{ + root: root, + } +} + +// ID is used for generating transfer ID by combining source and target ID. +// We want to keep the same rules for import from or export to local files +// no matter exact path, therefore it returns constant. The same as EML. +func (p *EMLProvider) ID() string { + return "local" //nolint[goconst] +} + +// Mailboxes returns all available folder names from root of EML files. +// In case the same folder name is used more than once (for example root/a/foo +// and root/b/foo), it's treated as the same folder. +func (p *EMLProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) { + var folderNames []string + var err error + if includeEmpty { + folderNames, err = getFolderNames(p.root) + } else { + folderNames, err = getFolderNamesWithFileSuffix(p.root, ".eml") + } + if err != nil { + return nil, err + } + + mailboxes := []Mailbox{} + for _, folderName := range folderNames { + mailboxes = append(mailboxes, Mailbox{ + ID: "", + Name: folderName, + Color: "", + IsExclusive: false, + }) + } + + return mailboxes, nil +} diff --git a/internal/transfer/provider_eml_source.go b/internal/transfer/provider_eml_source.go new file mode 100644 index 00000000..7eb949b8 --- /dev/null +++ b/internal/transfer/provider_eml_source.go @@ -0,0 +1,135 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "io/ioutil" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// TransferTo exports messages based on rules to channel. +func (p *EMLProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) { + log.Info("Started transfer from EML to channel") + defer log.Info("Finished transfer from EML to channel") + + filePathsPerFolder, err := p.getFilePathsPerFolder(rules) + if err != nil { + progress.fatal(err) + return + } + + // This list is not filtered by time but instead going throgh each file + // twice or keeping all in memory we will tell rough estimation which + // will be updated during processing each file. + for folderName, filePaths := range filePathsPerFolder { + if progress.shouldStop() { + break + } + + progress.updateCount(folderName, uint(len(filePaths))) + } + + for folderName, filePaths := range filePathsPerFolder { + // No error guaranteed by getFilePathsPerFolder. + rule, _ := rules.getRuleBySourceMailboxName(folderName) + log.WithField("rule", rule).Debug("Processing rule") + p.exportMessages(rule, filePaths, progress, ch) + } +} + +func (p *EMLProvider) getFilePathsPerFolder(rules transferRules) (map[string][]string, error) { + filePaths, err := getFilePathsWithSuffix(p.root, ".eml") + if err != nil { + return nil, err + } + + filePathsMap := map[string][]string{} + for _, filePath := range filePaths { + folder := filepath.Base(filepath.Dir(filepath.Join(p.root, filePath))) + _, err := rules.getRuleBySourceMailboxName(folder) + if err != nil { + log.WithField("msg", filePath).Trace("Message skipped due to folder name") + continue + } + + filePathsMap[folder] = append(filePathsMap[folder], filePath) + } + + return filePathsMap, nil +} + +func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *Progress, ch chan<- Message) { + count := uint(len(filePaths)) + + for _, filePath := range filePaths { + if progress.shouldStop() { + break + } + + msg, err := p.exportMessage(rule, filePath) + + // Read and check time in body only if the rule specifies it + // to not waste energy. + if err == nil && rule.HasTimeLimit() { + msgTime, msgTimeErr := getMessageTime(msg.Body) + if msgTimeErr != nil { + err = msgTimeErr + } else if !rule.isTimeInRange(msgTime) { + log.WithField("msg", filePath).Debug("Message skipped due to time") + + count-- + progress.updateCount(rule.SourceMailbox.Name, count) + continue + } + } + + // addMessage is called after time check to not report message + // which should not be exported but any error from reading body + // or parsing time is reported as an error. + progress.addMessage(filePath, rule) + progress.messageExported(filePath, msg.Body, err) + if err == nil { + ch <- msg + } + } +} + +func (p *EMLProvider) exportMessage(rule *Rule, filePath string) (Message, error) { + fullFilePath := filepath.Clean(filepath.Join(p.root, filePath)) + file, err := os.Open(fullFilePath) //nolint[gosec] + if err != nil { + return Message{}, errors.Wrap(err, "failed to open message") + } + defer file.Close() //nolint[errcheck] + + body, err := ioutil.ReadAll(file) + if err != nil { + return Message{}, errors.Wrap(err, "failed to read message") + } + + return Message{ + ID: filePath, + Unread: false, + Body: body, + Source: rule.SourceMailbox, + Targets: rule.TargetMailboxes, + }, nil +} diff --git a/internal/transfer/provider_eml_target.go b/internal/transfer/provider_eml_target.go new file mode 100644 index 00000000..58f7b9ac --- /dev/null +++ b/internal/transfer/provider_eml_target.go @@ -0,0 +1,89 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/hashicorp/go-multierror" +) + +// DefaultMailboxes returns the default mailboxes for default rules if no other is found. +func (p *EMLProvider) DefaultMailboxes(sourceMailbox Mailbox) []Mailbox { + return []Mailbox{{ + Name: sourceMailbox.Name, + }} +} + +// CreateMailbox does nothing. Folders are created dynamically during the import. +func (p *EMLProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) { + return mailbox, nil +} + +// TransferFrom imports messages from channel. +func (p *EMLProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) { + log.Info("Started transfer from channel to EML") + defer log.Info("Finished transfer from channel to EML") + + err := p.createFolders(rules) + if err != nil { + progress.fatal(err) + return + } + + for msg := range ch { + for progress.shouldStop() { + break + } + + err := p.writeFile(msg) + progress.messageImported(msg.ID, "", err) + } +} + +func (p *EMLProvider) createFolders(rules transferRules) error { + for rule := range rules.iterateActiveRules() { + for _, mailbox := range rule.TargetMailboxes { + path := filepath.Join(p.root, mailbox.Name) + if err := os.MkdirAll(path, os.ModePerm); err != nil { + return err + } + } + } + return nil +} + +func (p *EMLProvider) writeFile(msg Message) error { + fileName := filepath.Base(msg.ID) + if !strings.HasSuffix(fileName, ".eml") { + fileName += ".eml" + } + + var err error + for _, mailbox := range msg.Targets { + path := filepath.Join(p.root, mailbox.Name, fileName) + + if localErr := ioutil.WriteFile(path, msg.Body, 0600); localErr != nil { + err = multierror.Append(err, localErr) + } + } + return err +} diff --git a/internal/transfer/provider_eml_test.go b/internal/transfer/provider_eml_test.go new file mode 100644 index 00000000..76f499b6 --- /dev/null +++ b/internal/transfer/provider_eml_test.go @@ -0,0 +1,126 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + "time" + + r "github.com/stretchr/testify/require" +) + +func newTestEMLProvider(path string) *EMLProvider { + if path == "" { + path = "testdata/eml" + } + return NewEMLProvider(path) +} + +func TestEMLProviderMailboxes(t *testing.T) { + provider := newTestEMLProvider("") + + tests := []struct { + includeEmpty bool + wantMailboxes []Mailbox + }{ + {true, []Mailbox{ + {Name: "Foo"}, + {Name: "Inbox"}, + {Name: "eml"}, + }}, + {false, []Mailbox{ + {Name: "Foo"}, + {Name: "Inbox"}, + }}, + } + for _, tc := range tests { + tc := tc + t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) { + mailboxes, err := provider.Mailboxes(tc.includeEmpty, false) + r.NoError(t, err) + r.Equal(t, tc.wantMailboxes, mailboxes) + }) + } +} + +func TestEMLProviderTransferTo(t *testing.T) { + provider := newTestEMLProvider("") + + rules, rulesClose := newTestRules(t) + defer rulesClose() + setupEMLRules(rules) + + testTransferTo(t, rules, provider, []string{ + "Foo/msg.eml", + "Inbox/msg.eml", + }) +} + +func TestEMLProviderTransferFrom(t *testing.T) { + dir, err := ioutil.TempDir("", "eml") + r.NoError(t, err) + defer os.RemoveAll(dir) //nolint[errcheck] + + provider := newTestEMLProvider(dir) + + rules, rulesClose := newTestRules(t) + defer rulesClose() + setupEMLRules(rules) + + testTransferFrom(t, rules, provider, []Message{ + {ID: "Foo/msg.eml", Body: getTestMsgBody("msg"), Targets: []Mailbox{{Name: "Foo"}}}, + }) + + checkEMLFileStructure(t, dir, []string{ + "Foo/msg.eml", + }) +} + +func TestEMLProviderTransferFromTo(t *testing.T) { + dir, err := ioutil.TempDir("", "eml") + r.NoError(t, err) + defer os.RemoveAll(dir) //nolint[errcheck] + + source := newTestEMLProvider("") + target := newTestEMLProvider(dir) + + rules, rulesClose := newTestRules(t) + defer rulesClose() + setupEMLRules(rules) + + testTransferFromTo(t, rules, source, target, 5*time.Second) + + checkEMLFileStructure(t, dir, []string{ + "Foo/msg.eml", + "Inbox/msg.eml", + }) +} + +func setupEMLRules(rules transferRules) { + _ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0) + _ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0) +} + +func checkEMLFileStructure(t *testing.T, root string, expectedFiles []string) { + files, err := getFilePathsWithSuffix(root, ".eml") + r.NoError(t, err) + r.Equal(t, expectedFiles, files) +} diff --git a/internal/transfer/provider_imap.go b/internal/transfer/provider_imap.go new file mode 100644 index 00000000..16e7d493 --- /dev/null +++ b/internal/transfer/provider_imap.go @@ -0,0 +1,98 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "net" + "strings" + + imapClient "github.com/emersion/go-imap/client" +) + +// IMAPProvider implements export from IMAP server. +type IMAPProvider struct { + username string + password string + addr string + + client *imapClient.Client +} + +// NewIMAPProvider returns new IMAPProvider. +func NewIMAPProvider(username, password, host, port string) (*IMAPProvider, error) { + p := &IMAPProvider{ + username: username, + password: password, + addr: net.JoinHostPort(host, port), + } + + if err := p.auth(); err != nil { + return nil, err + } + + return p, nil +} + +// ID is used for generating transfer ID by combining source and target ID. +// We want to keep the same rules for import from any IMAP server, therefore +// it returns constant. +func (p *IMAPProvider) ID() string { + return "imap" +} + +// Mailboxes returns all available folder names from root of EML files. +// In case the same folder name is used more than once (for example root/a/foo +// and root/b/foo), it's treated as the same folder. +func (p *IMAPProvider) Mailboxes(includEmpty, includeAllMail bool) ([]Mailbox, error) { + mailboxesInfo, err := p.list() + if err != nil { + return nil, err + } + + mailboxes := []Mailbox{} + for _, mailbox := range mailboxesInfo { + hasNoSelect := false + for _, attrib := range mailbox.Attributes { + if strings.ToLower(attrib) == "\\noselect" { + hasNoSelect = true + break + } + } + if hasNoSelect || mailbox.Name == "[Gmail]" { + continue + } + + if !includEmpty || true { + mailboxStatus, err := p.selectIn(mailbox.Name) + if err != nil { + return nil, err + } + if mailboxStatus.Messages == 0 { + continue + } + } + + mailboxes = append(mailboxes, Mailbox{ + ID: "", + Name: mailbox.Name, + Color: "", + IsExclusive: false, + }) + } + return mailboxes, nil +} diff --git a/internal/transfer/provider_imap_source.go b/internal/transfer/provider_imap_source.go new file mode 100644 index 00000000..59dbf9ac --- /dev/null +++ b/internal/transfer/provider_imap_source.go @@ -0,0 +1,210 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "fmt" + "io/ioutil" + + "github.com/emersion/go-imap" +) + +type imapMessageInfo struct { + id string + uid uint32 + size uint32 +} + +const ( + imapPageSize = uint32(2000) // Optimized on Gmail. + imapMaxFetchSize = uint32(50 * 1000 * 1000) // Size in octets. If 0, it will use one fetch per message. +) + +// TransferTo exports messages based on rules to channel. +func (p *IMAPProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) { + log.Info("Started transfer from IMAP to channel") + defer log.Info("Finished transfer from IMAP to channel") + + imapMessageInfoMap := p.loadMessageInfoMap(rules, progress) + + for rule := range rules.iterateActiveRules() { + log.WithField("rule", rule).Debug("Processing rule") + messagesInfo := imapMessageInfoMap[rule.SourceMailbox.Name] + p.transferTo(rule, messagesInfo, progress, ch) + } +} + +func (p *IMAPProvider) loadMessageInfoMap(rules transferRules, progress *Progress) map[string]map[string]imapMessageInfo { + res := map[string]map[string]imapMessageInfo{} + + for rule := range rules.iterateActiveRules() { + if progress.shouldStop() { + break + } + + mailboxName := rule.SourceMailbox.Name + var mailbox *imap.MailboxStatus + progress.callWrap(func() error { + var err error + mailbox, err = p.selectIn(mailboxName) + return err + }) + if mailbox.Messages == 0 { + continue + } + + messagesInfo := p.loadMessagesInfo(rule, progress, mailbox.UidValidity) + res[rule.SourceMailbox.Name] = messagesInfo + progress.updateCount(rule.SourceMailbox.Name, uint(len(messagesInfo))) + } + + return res +} + +func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValidity uint32) map[string]imapMessageInfo { + messagesInfo := map[string]imapMessageInfo{} + + pageStart := uint32(1) + pageEnd := imapPageSize + for { + if progress.shouldStop() { + break + } + + seqSet := &imap.SeqSet{} + seqSet.AddRange(pageStart, pageEnd) + + items := []imap.FetchItem{imap.FetchUid, imap.FetchRFC822Size} + if rule.HasTimeLimit() { + items = append(items, imap.FetchEnvelope) + } + + pageMsgCount := uint32(0) + processMessageCallback := func(imapMessage *imap.Message) { + pageMsgCount++ + if rule.HasTimeLimit() { + t := imapMessage.Envelope.Date.Unix() + if t != 0 && !rule.isTimeInRange(t) { + log.WithField("uid", imapMessage.Uid).Debug("Message skipped due to time") + return + } + } + id := fmt.Sprintf("%s_%d:%d", rule.SourceMailbox.Name, uidValidity, imapMessage.Uid) + messagesInfo[id] = imapMessageInfo{ + id: id, + uid: imapMessage.Uid, + size: imapMessage.Size, + } + progress.addMessage(id, rule) + } + + progress.callWrap(func() error { + return p.fetch(seqSet, items, processMessageCallback) + }) + + if pageMsgCount < imapPageSize { + break + } + + pageStart = pageEnd + pageEnd += imapPageSize + } + + return messagesInfo +} + +func (p *IMAPProvider) transferTo(rule *Rule, messagesInfo map[string]imapMessageInfo, progress *Progress, ch chan<- Message) { + progress.callWrap(func() error { + _, err := p.selectIn(rule.SourceMailbox.Name) + return err + }) + + seqSet := &imap.SeqSet{} + seqSetSize := uint32(0) + uidToID := map[uint32]string{} + + for _, messageInfo := range messagesInfo { + if progress.shouldStop() { + break + } + + if seqSetSize != 0 && (seqSetSize+messageInfo.size) > imapMaxFetchSize { + log.WithField("mailbox", rule.SourceMailbox.Name).WithField("seq", seqSet).WithField("size", seqSetSize).Debug("Fetching messages") + + p.exportMessages(rule, progress, ch, seqSet, uidToID) + + seqSet = &imap.SeqSet{} + seqSetSize = 0 + uidToID = map[uint32]string{} + } + + seqSet.AddNum(messageInfo.uid) + seqSetSize += messageInfo.size + uidToID[messageInfo.uid] = messageInfo.id + } +} + +func (p *IMAPProvider) exportMessages(rule *Rule, progress *Progress, ch chan<- Message, seqSet *imap.SeqSet, uidToID map[uint32]string) { + section := &imap.BodySectionName{} + items := []imap.FetchItem{imap.FetchUid, imap.FetchFlags, section.FetchItem()} + + processMessageCallback := func(imapMessage *imap.Message) { + id, ok := uidToID[imapMessage.Uid] + + // Sometimes, server sends not requested messages. + if !ok { + log.WithField("uid", imapMessage.Uid).Warning("Message skipped: not requested") + return + } + + // Sometimes, server sends message twice, once with body and once without it. + bodyReader := imapMessage.GetBody(section) + if bodyReader == nil { + log.WithField("uid", imapMessage.Uid).Warning("Message skipped: no body") + return + } + + body, err := ioutil.ReadAll(bodyReader) + progress.messageExported(id, body, err) + if err == nil { + msg := p.exportMessage(rule, id, imapMessage, body) + ch <- msg + } + } + + progress.callWrap(func() error { + return p.uidFetch(seqSet, items, processMessageCallback) + }) +} + +func (p *IMAPProvider) exportMessage(rule *Rule, id string, imapMessage *imap.Message, body []byte) Message { + unread := true + for _, flag := range imapMessage.Flags { + if flag == imap.SeenFlag { + unread = false + } + } + + return Message{ + ID: id, + Unread: unread, + Body: body, + Source: rule.SourceMailbox, + Targets: rule.TargetMailboxes, + } +} diff --git a/internal/transfer/provider_imap_utils.go b/internal/transfer/provider_imap_utils.go new file mode 100644 index 00000000..364e9ecc --- /dev/null +++ b/internal/transfer/provider_imap_utils.go @@ -0,0 +1,236 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "net" + "time" + + imapID "github.com/ProtonMail/go-imap-id" + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/emersion/go-imap" + imapClient "github.com/emersion/go-imap/client" + sasl "github.com/emersion/go-sasl" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + imapDialTimeout = 5 * time.Second + imapRetries = 10 + imapReconnectTimeout = 30 * time.Minute + imapReconnectSleep = time.Minute +) + +type imapErrorLogger struct { + log *logrus.Entry +} + +func (l *imapErrorLogger) Printf(f string, v ...interface{}) { + l.log.Errorf(f, v...) +} + +func (l *imapErrorLogger) Println(v ...interface{}) { + l.log.Errorln(v...) +} + +type imapDebugLogger struct { //nolint[unused] + log *logrus.Entry +} + +func (l *imapDebugLogger) Write(data []byte) (int, error) { + l.log.Trace(string(data)) + return len(data), nil +} + +func (p *IMAPProvider) ensureConnection(callback func() error) error { + var callErr error + for i := 1; i <= imapRetries; i++ { + callErr = callback() + if callErr == nil { + return nil + } + + log.WithField("attempt", i).WithError(callErr).Warning("Call failed, trying reconnect") + err := p.tryReconnect() + if err != nil { + return err + } + } + return errors.Wrap(callErr, "too many retries") +} + +func (p *IMAPProvider) tryReconnect() error { + start := time.Now() + var previousErr error + for { + if time.Since(start) > imapReconnectTimeout { + return previousErr + } + + err := pmapi.CheckConnection() + if err != nil { + time.Sleep(imapReconnectSleep) + previousErr = err + continue + } + + err = p.reauth() + if err != nil { + time.Sleep(imapReconnectSleep) + previousErr = err + continue + } + + break + } + return nil +} + +func (p *IMAPProvider) reauth() error { + if _, err := p.client.Capability(); err != nil { + state := p.client.State() + log.WithField("addr", p.addr).WithField("state", state).WithError(err).Debug("Reconnecting") + p.client = nil + } + + return p.auth() +} + +func (p *IMAPProvider) auth() error { //nolint[funlen] + log := log.WithField("addr", p.addr) + + log.Info("Connecting to server") + + if _, err := net.DialTimeout("tcp", p.addr, imapDialTimeout); err != nil { + return errors.Wrap(err, "failed to dial server") + } + + client, err := imapClient.DialTLS(p.addr, nil) + if err != nil { + return errors.Wrap(err, "failed to connect to server") + } + client.ErrorLog = &imapErrorLogger{logrus.WithField("pkg", "imap-client")} + // Logrus have Writer helper but it fails for big messages because of + // bufio.MaxScanTokenSize limit. + // This spams a lot, uncomment once needed during development. + //client.SetDebug(&imapDebugLogger{logrus.WithField("pkg", "imap-client")}) + p.client = client + + log.Info("Connected") + + if (p.client.State() & imap.AuthenticatedState) == imap.AuthenticatedState { + return nil + } + + capability, err := p.client.Capability() + log.WithField("capability", capability).WithError(err).Debug("Server capability") + if err != nil { + return errors.Wrap(err, "failed to get capabilities") + } + + // SASL AUTH PLAIN + if ok, _ := p.client.SupportAuth("PLAIN"); p.client.State() == imap.NotAuthenticatedState && ok { + log.Debug("Trying plain auth") + authPlain := sasl.NewPlainClient("", p.username, p.password) + if err = p.client.Authenticate(authPlain); err != nil { + return errors.Wrap(err, "plain auth failed") + } + } + + // LOGIN: if the server reports the IMAP4rev1 capability then it is standards conformant and must support login. + if ok, _ := p.client.Support("IMAP4rev1"); p.client.State() == imap.NotAuthenticatedState && ok { + log.Debug("Trying login") + if err = p.client.Login(p.username, p.password); err != nil { + return errors.Wrap(err, "login failed") + } + } + + if p.client.State() == imap.NotAuthenticatedState { + return errors.New("unknown auth method") + } + + log.Info("Logged in") + + idClient := imapID.NewClient(p.client) + if ok, err := idClient.SupportID(); err == nil && ok { + serverID, err := idClient.ID(imapID.ID{ + imapID.FieldName: "ImportExport", + imapID.FieldVersion: "beta", + }) + log.WithField("ID", serverID).WithError(err).Debug("Server info") + } + + return err +} + +func (p *IMAPProvider) list() (mailboxes []*imap.MailboxInfo, err error) { + err = p.ensureConnection(func() error { + mailboxesCh := make(chan *imap.MailboxInfo) + doneCh := make(chan error) + + go func() { + doneCh <- p.client.List("", "*", mailboxesCh) + }() + + for mailbox := range mailboxesCh { + mailboxes = append(mailboxes, mailbox) + } + + return <-doneCh + }) + return +} + +func (p *IMAPProvider) selectIn(mailboxName string) (mailbox *imap.MailboxStatus, err error) { + err = p.ensureConnection(func() error { + mailbox, err = p.client.Select(mailboxName, true) + return err + }) + return +} + +func (p *IMAPProvider) fetch(seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error { + return p.fetchHelper(false, seqSet, items, processMessageCallback) +} + +func (p *IMAPProvider) uidFetch(seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error { + return p.fetchHelper(true, seqSet, items, processMessageCallback) +} + +func (p *IMAPProvider) fetchHelper(uid bool, seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error { + return p.ensureConnection(func() error { + messagesCh := make(chan *imap.Message) + doneCh := make(chan error) + + go func() { + if uid { + doneCh <- p.client.UidFetch(seqSet, items, messagesCh) + } else { + doneCh <- p.client.Fetch(seqSet, items, messagesCh) + } + }() + + for message := range messagesCh { + processMessageCallback(message) + } + + err := <-doneCh + return err + }) +} diff --git a/internal/transfer/provider_local.go b/internal/transfer/provider_local.go new file mode 100644 index 00000000..4b52064d --- /dev/null +++ b/internal/transfer/provider_local.go @@ -0,0 +1,68 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +// LocalProvider implements import from local EML and MBOX file structure. +type LocalProvider struct { + root string + emlProvider *EMLProvider + mboxProvider *MBOXProvider +} + +func NewLocalProvider(root string) *LocalProvider { + return &LocalProvider{ + root: root, + emlProvider: NewEMLProvider(root), + mboxProvider: NewMBOXProvider(root), + } +} + +// ID is used for generating transfer ID by combining source and target ID. +// We want to keep the same rules for import from or export to local files +// no matter exact path, therefore it returns constant. +// The same as EML and MBOX. +func (p *LocalProvider) ID() string { + return "local" //nolint[goconst] +} + +// Mailboxes returns all available folder names from root of EML and MBOX files. +func (p *LocalProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) { + mailboxes, err := p.emlProvider.Mailboxes(includeEmpty, includeAllMail) + if err != nil { + return nil, err + } + + mboxMailboxes, err := p.mboxProvider.Mailboxes(includeEmpty, includeAllMail) + if err != nil { + return nil, err + } + + for _, mboxMailbox := range mboxMailboxes { + found := false + for _, mailboxes := range mailboxes { + if mboxMailbox.Name == mailboxes.Name { + found = true + break + } + } + if !found { + mailboxes = append(mailboxes, mboxMailbox) + } + } + return mailboxes, nil +} diff --git a/internal/transfer/provider_local_source.go b/internal/transfer/provider_local_source.go new file mode 100644 index 00000000..e3549fe8 --- /dev/null +++ b/internal/transfer/provider_local_source.go @@ -0,0 +1,42 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "sync" +) + +// TransferTo exports messages based on rules to channel. +func (p *LocalProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) { + log.Info("Started transfer from EML and MBOX to channel") + defer log.Info("Finished transfer from EML and MBOX to channel") + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + p.emlProvider.TransferTo(rules, progress, ch) + }() + go func() { + defer wg.Done() + p.mboxProvider.TransferTo(rules, progress, ch) + }() + + wg.Wait() +} diff --git a/internal/transfer/provider_local_test.go b/internal/transfer/provider_local_test.go new file mode 100644 index 00000000..8cd9093b --- /dev/null +++ b/internal/transfer/provider_local_test.go @@ -0,0 +1,77 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "fmt" + "testing" + + r "github.com/stretchr/testify/require" +) + +func newTestLocalProvider(path string) *LocalProvider { + if path == "" { + path = "testdata/emlmbox" + } + return NewLocalProvider(path) +} + +func TestLocalProviderMailboxes(t *testing.T) { + provider := newTestLocalProvider("") + + tests := []struct { + includeEmpty bool + wantMailboxes []Mailbox + }{ + {true, []Mailbox{ + {Name: "Foo"}, + {Name: "emlmbox"}, + {Name: "Inbox"}, + }}, + {false, []Mailbox{ + {Name: "Foo"}, + {Name: "Inbox"}, + }}, + } + for _, tc := range tests { + tc := tc + t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) { + mailboxes, err := provider.Mailboxes(tc.includeEmpty, false) + r.NoError(t, err) + r.Equal(t, tc.wantMailboxes, mailboxes) + }) + } +} + +func TestLocalProviderTransferTo(t *testing.T) { + provider := newTestLocalProvider("") + + rules, rulesClose := newTestRules(t) + defer rulesClose() + setupEMLMBOXRules(rules) + + testTransferTo(t, rules, provider, []string{ + "Foo/msg.eml", + "Inbox.mbox:1", + }) +} + +func setupEMLMBOXRules(rules transferRules) { + _ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0) + _ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0) +} diff --git a/internal/transfer/provider_mbox.go b/internal/transfer/provider_mbox.go new file mode 100644 index 00000000..0156fbc1 --- /dev/null +++ b/internal/transfer/provider_mbox.go @@ -0,0 +1,66 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "path/filepath" + "strings" +) + +// MBOXProvider implements import and export to/from MBOX structure. +type MBOXProvider struct { + root string +} + +func NewMBOXProvider(root string) *MBOXProvider { + return &MBOXProvider{ + root: root, + } +} + +// ID is used for generating transfer ID by combining source and target ID. +// We want to keep the same rules for import from or export to local files +// no matter exact path, therefore it returns constant. The same as EML. +func (p *MBOXProvider) ID() string { + return "local" //nolint[goconst] +} + +// Mailboxes returns all available folder names from root of EML files. +// In case the same folder name is used more than once (for example root/a/foo +// and root/b/foo), it's treated as the same folder. +func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) { + filePaths, err := getFilePathsWithSuffix(p.root, "mbox") + if err != nil { + return nil, err + } + + mailboxes := []Mailbox{} + for _, filePath := range filePaths { + fileName := filepath.Base(filePath) + mailboxName := strings.TrimSuffix(fileName, ".mbox") + + mailboxes = append(mailboxes, Mailbox{ + ID: "", + Name: mailboxName, + Color: "", + IsExclusive: false, + }) + } + + return mailboxes, nil +} diff --git a/internal/transfer/provider_mbox_source.go b/internal/transfer/provider_mbox_source.go new file mode 100644 index 00000000..68491893 --- /dev/null +++ b/internal/transfer/provider_mbox_source.go @@ -0,0 +1,183 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/emersion/go-mbox" + "github.com/pkg/errors" +) + +// TransferTo exports messages based on rules to channel. +func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) { + log.Info("Started transfer from MBOX to channel") + defer log.Info("Finished transfer from MBOX to channel") + + filePathsPerFolder, err := p.getFilePathsPerFolder(rules) + if err != nil { + progress.fatal(err) + return + } + + for folderName, filePaths := range filePathsPerFolder { + // No error guaranteed by getFilePathsPerFolder. + rule, _ := rules.getRuleBySourceMailboxName(folderName) + for _, filePath := range filePaths { + if progress.shouldStop() { + break + } + p.updateCount(rule, progress, filePath) + } + } + + for folderName, filePaths := range filePathsPerFolder { + // No error guaranteed by getFilePathsPerFolder. + rule, _ := rules.getRuleBySourceMailboxName(folderName) + log.WithField("rule", rule).Debug("Processing rule") + for _, filePath := range filePaths { + if progress.shouldStop() { + break + } + p.transferTo(rule, progress, ch, filePath) + } + } +} + +func (p *MBOXProvider) getFilePathsPerFolder(rules transferRules) (map[string][]string, error) { + filePaths, err := getFilePathsWithSuffix(p.root, ".mbox") + if err != nil { + return nil, err + } + + filePathsMap := map[string][]string{} + for _, filePath := range filePaths { + fileName := filepath.Base(filePath) + folder := strings.TrimSuffix(fileName, ".mbox") + _, err := rules.getRuleBySourceMailboxName(folder) + if err != nil { + log.WithField("msg", filePath).Trace("Mailbox skipped due to folder name") + continue + } + + filePathsMap[folder] = append(filePathsMap[folder], filePath) + } + return filePathsMap, nil +} + +func (p *MBOXProvider) updateCount(rule *Rule, progress *Progress, filePath string) { + mboxReader := p.openMbox(progress, filePath) + if mboxReader == nil { + return + } + + count := 0 + for { + _, err := mboxReader.NextMessage() + if err != nil { + break + } + count++ + } + progress.updateCount(rule.SourceMailbox.Name, uint(count)) +} + +func (p *MBOXProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, filePath string) { + mboxReader := p.openMbox(progress, filePath) + if mboxReader == nil { + return + } + + index := 0 + count := 0 + for { + if progress.shouldStop() { + break + } + + index++ + id := fmt.Sprintf("%s:%d", filePath, index) + + msgReader, err := mboxReader.NextMessage() + if err == io.EOF { + break + } else if err != nil { + progress.fatal(err) + break + } + + msg, err := p.exportMessage(rule, id, msgReader) + + // Read and check time in body only if the rule specifies it + // to not waste energy. + if err == nil && rule.HasTimeLimit() { + msgTime, msgTimeErr := getMessageTime(msg.Body) + if msgTimeErr != nil { + err = msgTimeErr + } else if !rule.isTimeInRange(msgTime) { + log.WithField("msg", id).Debug("Message skipped due to time") + continue + } + } + + // Counting only messages filtered by time to update count to correct total. + count++ + + // addMessage is called after time check to not report message + // which should not be exported but any error from reading body + // or parsing time is reported as an error. + progress.addMessage(id, rule) + progress.messageExported(id, msg.Body, err) + if err == nil { + ch <- msg + } + } + progress.updateCount(rule.SourceMailbox.Name, uint(count)) +} + +func (p *MBOXProvider) exportMessage(rule *Rule, id string, msgReader io.Reader) (Message, error) { + body, err := ioutil.ReadAll(msgReader) + if err != nil { + return Message{}, errors.Wrap(err, "failed to read message") + } + + return Message{ + ID: id, + Unread: false, + Body: body, + Source: rule.SourceMailbox, + Targets: rule.TargetMailboxes, + }, nil +} + +func (p *MBOXProvider) openMbox(progress *Progress, mboxPath string) *mbox.Reader { + mboxPath = filepath.Join(p.root, mboxPath) + mboxFile, err := os.Open(mboxPath) //nolint[gosec] + if os.IsNotExist(err) { + return nil + } else if err != nil { + progress.fatal(err) + return nil + } + return mbox.NewReader(mboxFile) +} diff --git a/internal/transfer/provider_mbox_target.go b/internal/transfer/provider_mbox_target.go new file mode 100644 index 00000000..610f8cf2 --- /dev/null +++ b/internal/transfer/provider_mbox_target.go @@ -0,0 +1,97 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "os" + "path/filepath" + "strings" + "time" + + "github.com/emersion/go-mbox" + "github.com/hashicorp/go-multierror" +) + +// DefaultMailboxes returns the default mailboxes for default rules if no other is found. +func (p *MBOXProvider) DefaultMailboxes(sourceMailbox Mailbox) []Mailbox { + return []Mailbox{{ + Name: sourceMailbox.Name, + }} +} + +// CreateMailbox does nothing. Files are created dynamically during the import. +func (p *MBOXProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) { + return mailbox, nil +} + +// TransferFrom imports messages from channel. +func (p *MBOXProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) { + log.Info("Started transfer from channel to MBOX") + defer log.Info("Finished transfer from channel to MBOX") + + for msg := range ch { + for progress.shouldStop() { + break + } + + err := p.writeMessage(msg) + progress.messageImported(msg.ID, "", err) + } +} + +func (p *MBOXProvider) writeMessage(msg Message) error { + var multiErr error + for _, mailbox := range msg.Targets { + mboxName := filepath.Base(mailbox.Name) + if !strings.HasSuffix(mboxName, ".mbox") { + mboxName += ".mbox" + } + + mboxPath := filepath.Join(p.root, mboxName) + mboxFile, err := os.OpenFile(mboxPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + multiErr = multierror.Append(multiErr, err) + continue + } + + msgFrom := "" + msgTime := time.Now() + if header, err := getMessageHeader(msg.Body); err == nil { + if date, err := header.Date(); err == nil { + msgTime = date + } + if addresses, err := header.AddressList("from"); err == nil && len(addresses) > 0 { + msgFrom = addresses[0].Address + } + } + + mboxWriter := mbox.NewWriter(mboxFile) + messageWriter, err := mboxWriter.CreateMessage(msgFrom, msgTime) + if err != nil { + multiErr = multierror.Append(multiErr, err) + continue + } + + _, err = messageWriter.Write(msg.Body) + if err != nil { + multiErr = multierror.Append(multiErr, err) + continue + } + } + return multiErr +} diff --git a/internal/transfer/provider_mbox_test.go b/internal/transfer/provider_mbox_test.go new file mode 100644 index 00000000..d9145644 --- /dev/null +++ b/internal/transfer/provider_mbox_test.go @@ -0,0 +1,125 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + "time" + + r "github.com/stretchr/testify/require" +) + +func newTestMBOXProvider(path string) *MBOXProvider { + if path == "" { + path = "testdata/mbox" + } + return NewMBOXProvider(path) +} + +func TestMBOXProviderMailboxes(t *testing.T) { + provider := newTestMBOXProvider("") + + tests := []struct { + includeEmpty bool + wantMailboxes []Mailbox + }{ + {true, []Mailbox{ + {Name: "Foo"}, + {Name: "Inbox"}, + }}, + {false, []Mailbox{ + {Name: "Foo"}, + {Name: "Inbox"}, + }}, + } + for _, tc := range tests { + tc := tc + t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) { + mailboxes, err := provider.Mailboxes(tc.includeEmpty, false) + r.NoError(t, err) + r.Equal(t, tc.wantMailboxes, mailboxes) + }) + } +} + +func TestMBOXProviderTransferTo(t *testing.T) { + provider := newTestMBOXProvider("") + + rules, rulesClose := newTestRules(t) + defer rulesClose() + setupMBOXRules(rules) + + testTransferTo(t, rules, provider, []string{ + "Foo.mbox:1", + "Inbox.mbox:1", + }) +} + +func TestMBOXProviderTransferFrom(t *testing.T) { + dir, err := ioutil.TempDir("", "eml") + r.NoError(t, err) + defer os.RemoveAll(dir) //nolint[errcheck] + + provider := newTestMBOXProvider(dir) + + rules, rulesClose := newTestRules(t) + defer rulesClose() + setupMBOXRules(rules) + + testTransferFrom(t, rules, provider, []Message{ + {ID: "Foo.mbox:1", Body: getTestMsgBody("msg"), Targets: []Mailbox{{Name: "Foo"}}}, + }) + + checkMBOXFileStructure(t, dir, []string{ + "Foo.mbox", + }) +} + +func TestMBOXProviderTransferFromTo(t *testing.T) { + dir, err := ioutil.TempDir("", "eml") + r.NoError(t, err) + defer os.RemoveAll(dir) //nolint[errcheck] + + source := newTestMBOXProvider("") + target := newTestMBOXProvider(dir) + + rules, rulesClose := newTestRules(t) + defer rulesClose() + setupEMLRules(rules) + + testTransferFromTo(t, rules, source, target, 5*time.Second) + + checkMBOXFileStructure(t, dir, []string{ + "Foo.mbox", + "Inbox.mbox", + }) +} + +func setupMBOXRules(rules transferRules) { + _ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0) + _ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0) +} + +func checkMBOXFileStructure(t *testing.T, root string, expectedFiles []string) { + files, err := getFilePathsWithSuffix(root, ".mbox") + r.NoError(t, err) + r.Equal(t, expectedFiles, files) +} diff --git a/internal/transfer/provider_pmapi.go b/internal/transfer/provider_pmapi.go new file mode 100644 index 00000000..f6f14e27 --- /dev/null +++ b/internal/transfer/provider_pmapi.go @@ -0,0 +1,140 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "sort" + + "github.com/ProtonMail/gopenpgp/crypto" + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/pkg/errors" +) + +// PMAPIProvider implements import and export to/from ProtonMail server. +type PMAPIProvider struct { + clientManager ClientManager + userID string + addressID string + keyRing *crypto.KeyRing +} + +// NewPMAPIProvider returns new PMAPIProvider. +func NewPMAPIProvider(clientManager ClientManager, userID, addressID string) (*PMAPIProvider, error) { + keyRing, err := clientManager.GetClient(userID).KeyRingForAddressID(addressID) + if err != nil { + return nil, errors.Wrap(err, "failed to get key ring") + } + + return &PMAPIProvider{ + clientManager: clientManager, + userID: userID, + addressID: addressID, + keyRing: keyRing, + }, nil +} + +func (p *PMAPIProvider) client() pmapi.Client { + return p.clientManager.GetClient(p.userID) +} + +// ID returns identifier of current setup of PMAPI provider. +// Identification is unique per user. +func (p *PMAPIProvider) ID() string { + return p.userID +} + +// Mailboxes returns all available labels in ProtonMail account. +func (p *PMAPIProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) { + labels, err := p.client().ListLabels() + if err != nil { + return nil, err + } + sortedLabels := byFoldersLabels(labels) + sort.Sort(sortedLabels) + + emptyLabelsMap := map[string]bool{} + if !includeEmpty { + messagesCounts, err := p.client().CountMessages(p.addressID) + if err != nil { + return nil, err + } + for _, messagesCount := range messagesCounts { + if messagesCount.Total == 0 { + emptyLabelsMap[messagesCount.LabelID] = true + } + } + } + + mailboxes := getSystemMailboxes(includeAllMail) + for _, label := range sortedLabels { + if !includeEmpty && emptyLabelsMap[label.ID] { + continue + } + + mailboxes = append(mailboxes, Mailbox{ + ID: label.ID, + Name: label.Name, + Color: label.Color, + IsExclusive: label.Exclusive == 1, + }) + } + return mailboxes, nil +} + +func getSystemMailboxes(includeAllMail bool) []Mailbox { + mailboxes := []Mailbox{ + {ID: pmapi.InboxLabel, Name: "Inbox", IsExclusive: true}, + {ID: pmapi.DraftLabel, Name: "Drafts", IsExclusive: true}, + {ID: pmapi.SentLabel, Name: "Sent", IsExclusive: true}, + {ID: pmapi.StarredLabel, Name: "Starred", IsExclusive: true}, + {ID: pmapi.ArchiveLabel, Name: "Archive", IsExclusive: true}, + {ID: pmapi.SpamLabel, Name: "Spam", IsExclusive: true}, + {ID: pmapi.TrashLabel, Name: "Trash", IsExclusive: true}, + } + + if includeAllMail { + mailboxes = append(mailboxes, Mailbox{ + ID: pmapi.AllMailLabel, + Name: "All Mail", + IsExclusive: true, + }) + } + + return mailboxes +} + +type byFoldersLabels []*pmapi.Label + +func (l byFoldersLabels) Len() int { + return len(l) +} + +func (l byFoldersLabels) Swap(i, j int) { + l[i], l[j] = l[j], l[i] +} + +// Less sorts first folders, then labels, by user order. +func (l byFoldersLabels) Less(i, j int) bool { + if l[i].Exclusive == 1 && l[j].Exclusive == 0 { + return true + } + if l[i].Exclusive == 0 && l[j].Exclusive == 1 { + return false + } + return l[i].Order < l[j].Order +} diff --git a/internal/transfer/provider_pmapi_source.go b/internal/transfer/provider_pmapi_source.go new file mode 100644 index 00000000..240a4a84 --- /dev/null +++ b/internal/transfer/provider_pmapi_source.go @@ -0,0 +1,161 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "fmt" + + pkgMessage "github.com/ProtonMail/proton-bridge/pkg/message" + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/pkg/errors" +) + +const pmapiListPageSize = 150 + +// TransferTo exports messages based on rules to channel. +func (p *PMAPIProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) { + log.Info("Started transfer from PMAPI to channel") + defer log.Info("Finished transfer from PMAPI to channel") + + go p.loadCounts(rules, progress) + + for rule := range rules.iterateActiveRules() { + p.transferTo(rule, progress, ch, rules.skipEncryptedMessages) + } +} + +func (p *PMAPIProvider) loadCounts(rules transferRules, progress *Progress) { + for rule := range rules.iterateActiveRules() { + if progress.shouldStop() { + break + } + + rule := rule + progress.callWrap(func() error { + _, total, err := p.listMessages(&pmapi.MessagesFilter{ + LabelID: rule.SourceMailbox.ID, + Begin: rule.FromTime, + End: rule.ToTime, + Limit: 0, + }) + if err != nil { + log.WithError(err).Warning("Problem to load counts") + return err + } + progress.updateCount(rule.SourceMailbox.Name, uint(total)) + return nil + }) + } +} + +func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, skipEncryptedMessages bool) { + nextID := "" + for { + if progress.shouldStop() { + break + } + + isLastPage := true + + progress.callWrap(func() error { + desc := false + pmapiMessages, count, err := p.listMessages(&pmapi.MessagesFilter{ + LabelID: rule.SourceMailbox.ID, + Begin: rule.FromTime, + End: rule.ToTime, + BeginID: nextID, + PageSize: pmapiListPageSize, + Page: 0, + Sort: "ID", + Desc: &desc, + }) + if err != nil { + return err + } + log.WithField("label", rule.SourceMailbox.ID).WithField("next", nextID).WithField("count", count).Debug("Listing messages") + + isLastPage = len(pmapiMessages) < pmapiListPageSize + + // The first ID is the last one from the last page (= do not export twice the same one). + if nextID != "" { + pmapiMessages = pmapiMessages[1:] + } + + for _, pmapiMessage := range pmapiMessages { + if progress.shouldStop() { + break + } + + msgID := fmt.Sprintf("%s_%s", rule.SourceMailbox.ID, pmapiMessage.ID) + progress.addMessage(msgID, rule) + msg, err := p.exportMessage(rule, progress, pmapiMessage.ID, msgID, skipEncryptedMessages) + progress.messageExported(msgID, msg.Body, err) + if err == nil { + ch <- msg + } + } + + if !isLastPage { + nextID = pmapiMessages[len(pmapiMessages)-1].ID + } + + return nil + }) + + if isLastPage { + break + } + } +} + +func (p *PMAPIProvider) exportMessage(rule *Rule, progress *Progress, pmapiMsgID, msgID string, skipEncryptedMessages bool) (Message, error) { + var msg *pmapi.Message + progress.callWrap(func() error { + var err error + msg, err = p.getMessage(pmapiMsgID) + return err + }) + + msgBuilder := pkgMessage.NewBuilder(p.client(), msg) + msgBuilder.EncryptedToHTML = false + _, body, err := msgBuilder.BuildMessage() + if err != nil { + return Message{ + Body: body, // Keep body to show details about the message to user. + }, errors.Wrap(err, "failed to build message") + } + + if !msgBuilder.SuccessfullyDecrypted() && skipEncryptedMessages { + return Message{ + Body: body, // Keep body to show details about the message to user. + }, errors.New("skipping encrypted message") + } + + unread := false + if msg.Unread == 1 { + unread = true + } + + return Message{ + ID: msgID, + Unread: unread, + Body: body, + Source: rule.SourceMailbox, + Targets: rule.TargetMailboxes, + }, nil +} diff --git a/internal/transfer/provider_pmapi_target.go b/internal/transfer/provider_pmapi_target.go new file mode 100644 index 00000000..1034caa4 --- /dev/null +++ b/internal/transfer/provider_pmapi_target.go @@ -0,0 +1,220 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + + pkgMessage "github.com/ProtonMail/proton-bridge/pkg/message" + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/pkg/errors" +) + +// DefaultMailboxes returns the default mailboxes for default rules if no other is found. +func (p *PMAPIProvider) DefaultMailboxes(_ Mailbox) []Mailbox { + return []Mailbox{{ + ID: pmapi.ArchiveLabel, + Name: "Archive", + IsExclusive: true, + }} +} + +// CreateMailbox creates label in ProtonMail account. +func (p *PMAPIProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) { + if mailbox.ID != "" { + return Mailbox{}, errors.New("mailbox is already created") + } + + exclusive := 0 + if mailbox.IsExclusive { + exclusive = 1 + } + + label, err := p.client().CreateLabel(&pmapi.Label{ + Name: mailbox.Name, + Color: mailbox.Color, + Exclusive: exclusive, + Type: pmapi.LabelTypeMailbox, + }) + if err != nil { + return Mailbox{}, errors.Wrap(err, fmt.Sprintf("failed to create mailbox %s", mailbox.Name)) + } + mailbox.ID = label.ID + return mailbox, nil +} + +// TransferFrom imports messages from channel. +func (p *PMAPIProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) { + log.Info("Started transfer from channel to PMAPI") + defer log.Info("Finished transfer from channel to PMAPI") + + for msg := range ch { + for progress.shouldStop() { + break + } + + var importedID string + var err error + if p.isMessageDraft(msg) { + importedID, err = p.importDraft(msg, rules.globalMailbox) + } else { + importedID, err = p.importMessage(msg, rules.globalMailbox) + } + progress.messageImported(msg.ID, importedID, err) + } +} + +func (p *PMAPIProvider) isMessageDraft(msg Message) bool { + for _, target := range msg.Targets { + if target.ID == pmapi.DraftLabel { + return true + } + } + return false +} + +func (p *PMAPIProvider) importDraft(msg Message, globalMailbox *Mailbox) (string, error) { + message, attachmentReaders, err := p.parseMessage(msg) + if err != nil { + return "", errors.Wrap(err, "failed to parse message") + } + + if err := message.Encrypt(p.keyRing, nil); err != nil { + return "", errors.Wrap(err, "failed to encrypt draft") + } + + if globalMailbox != nil { + message.LabelIDs = append(message.LabelIDs, globalMailbox.ID) + } + + attachments := message.Attachments + message.Attachments = nil + + draft, err := p.createDraft(message, "", pmapi.DraftActionReply) + if err != nil { + return "", errors.Wrap(err, "failed to create draft") + } + + for idx, attachment := range attachments { + attachment.MessageID = draft.ID + attachmentBody, _ := ioutil.ReadAll(attachmentReaders[idx]) + + r := bytes.NewReader(attachmentBody) + sigReader, err := attachment.DetachedSign(p.keyRing, r) + if err != nil { + return "", errors.Wrap(err, "failed to sign attachment") + } + + r = bytes.NewReader(attachmentBody) + encReader, err := attachment.Encrypt(p.keyRing, r) + if err != nil { + return "", errors.Wrap(err, "failed to encrypt attachment") + } + + _, err = p.createAttachment(attachment, encReader, sigReader) + if err != nil { + return "", errors.Wrap(err, "failed to create attachment") + } + } + + return draft.ID, nil +} + +func (p *PMAPIProvider) importMessage(msg Message, globalMailbox *Mailbox) (string, error) { + message, attachmentReaders, err := p.parseMessage(msg) + if err != nil { + return "", errors.Wrap(err, "failed to parse message") + } + + body, err := p.encryptMessage(message, attachmentReaders) + if err != nil { + return "", errors.Wrap(err, "failed to encrypt message") + } + + unread := 0 + if msg.Unread { + unread = 1 + } + + labelIDs := []string{} + for _, target := range msg.Targets { + // Frontend should not set All Mail to Rules, but to be sure... + if target.ID != pmapi.AllMailLabel { + labelIDs = append(labelIDs, target.ID) + } + } + if globalMailbox != nil { + labelIDs = append(labelIDs, globalMailbox.ID) + } + + importMsgReq := &pmapi.ImportMsgReq{ + AddressID: p.addressID, + Body: body, + Unread: unread, + Time: message.Time, + Flags: computeMessageFlags(labelIDs), + LabelIDs: labelIDs, + } + + results, err := p.importRequest([]*pmapi.ImportMsgReq{importMsgReq}) + if err != nil { + return "", errors.Wrap(err, "failed to import messages") + } + if len(results) == 0 { + return "", errors.New("import ended with no result") + } + if results[0].Error != nil { + return "", errors.Wrap(results[0].Error, "failed to import message") + } + return results[0].MessageID, nil +} + +func (p *PMAPIProvider) parseMessage(msg Message) (*pmapi.Message, []io.Reader, error) { + message, _, _, attachmentReaders, err := pkgMessage.Parse(bytes.NewBuffer(msg.Body), "", "") + return message, attachmentReaders, err +} + +func (p *PMAPIProvider) encryptMessage(msg *pmapi.Message, attachmentReaders []io.Reader) ([]byte, error) { + if msg.MIMEType == pmapi.ContentTypeMultipartEncrypted { + return []byte(msg.Body), nil + } + return pkgMessage.BuildEncrypted(msg, attachmentReaders, p.keyRing) +} + +func computeMessageFlags(labels []string) (flag int64) { + for _, labelID := range labels { + switch labelID { + case pmapi.SentLabel: + flag = (flag | pmapi.FlagSent) + case pmapi.ArchiveLabel, pmapi.InboxLabel: + flag = (flag | pmapi.FlagReceived) + case pmapi.DraftLabel: + log.Error("Found draft target in non-draft import") + } + } + + // NOTE: if the labels are custom only + if flag == 0 { + flag = pmapi.FlagReceived + } + + return flag +} diff --git a/internal/transfer/provider_pmapi_test.go b/internal/transfer/provider_pmapi_test.go new file mode 100644 index 00000000..6d529fae --- /dev/null +++ b/internal/transfer/provider_pmapi_test.go @@ -0,0 +1,201 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "bytes" + "fmt" + "testing" + "time" + + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + gomock "github.com/golang/mock/gomock" + r "github.com/stretchr/testify/require" +) + +func TestPMAPIProviderMailboxes(t *testing.T) { + m := initMocks(t) + defer m.ctrl.Finish() + + setupPMAPIClientExpectationForExport(&m) + provider, err := NewPMAPIProvider(m.clientManager, "user", "addressID") + r.NoError(t, err) + + tests := []struct { + includeEmpty bool + includeAllMail bool + wantMailboxes []Mailbox + }{ + {true, false, []Mailbox{ + {ID: "folder1", Name: "One", Color: "red", IsExclusive: true}, + {ID: "folder2", Name: "Two", Color: "orange", IsExclusive: true}, + {ID: "label2", Name: "Bar", Color: "green", IsExclusive: false}, + {ID: "label1", Name: "Foo", Color: "blue", IsExclusive: false}, + }}, + {false, true, []Mailbox{ + {ID: pmapi.AllMailLabel, Name: "All Mail", IsExclusive: true}, + {ID: "folder1", Name: "One", Color: "red", IsExclusive: true}, + {ID: "folder2", Name: "Two", Color: "orange", IsExclusive: true}, + {ID: "label1", Name: "Foo", Color: "blue", IsExclusive: false}, + }}, + } + for _, tc := range tests { + tc := tc + t.Run(fmt.Sprintf("%v-%v", tc.includeEmpty, tc.includeAllMail), func(t *testing.T) { + mailboxes, err := provider.Mailboxes(tc.includeEmpty, tc.includeAllMail) + r.NoError(t, err) + r.Equal(t, []Mailbox{ + {ID: pmapi.InboxLabel, Name: "Inbox", IsExclusive: true}, + {ID: pmapi.DraftLabel, Name: "Drafts", IsExclusive: true}, + {ID: pmapi.SentLabel, Name: "Sent", IsExclusive: true}, + {ID: pmapi.StarredLabel, Name: "Starred", IsExclusive: true}, + {ID: pmapi.ArchiveLabel, Name: "Archive", IsExclusive: true}, + {ID: pmapi.SpamLabel, Name: "Spam", IsExclusive: true}, + {ID: pmapi.TrashLabel, Name: "Trash", IsExclusive: true}, + }, mailboxes[:7]) + r.Equal(t, tc.wantMailboxes, mailboxes[7:]) + }) + } +} + +func TestPMAPIProviderTransferTo(t *testing.T) { + m := initMocks(t) + defer m.ctrl.Finish() + + setupPMAPIClientExpectationForExport(&m) + provider, err := NewPMAPIProvider(m.clientManager, "user", "addressID") + r.NoError(t, err) + + rules, rulesClose := newTestRules(t) + defer rulesClose() + setupPMAPIRules(rules) + + testTransferTo(t, rules, provider, []string{ + "0_msg1", + "0_msg2", + }) +} + +func TestPMAPIProviderTransferFrom(t *testing.T) { + m := initMocks(t) + defer m.ctrl.Finish() + + setupPMAPIClientExpectationForImport(&m) + provider, err := NewPMAPIProvider(m.clientManager, "user", "addressID") + r.NoError(t, err) + + rules, rulesClose := newTestRules(t) + defer rulesClose() + setupPMAPIRules(rules) + + testTransferFrom(t, rules, provider, []Message{ + {ID: "msg1", Body: getTestMsgBody("msg1"), Targets: []Mailbox{{ID: pmapi.InboxLabel}}}, + {ID: "msg2", Body: getTestMsgBody("msg2"), Targets: []Mailbox{{ID: pmapi.InboxLabel}}}, + }) +} + +func TestPMAPIProviderTransferFromDraft(t *testing.T) { + m := initMocks(t) + defer m.ctrl.Finish() + + setupPMAPIClientExpectationForImportDraft(&m) + provider, err := NewPMAPIProvider(m.clientManager, "user", "addressID") + r.NoError(t, err) + + rules, rulesClose := newTestRules(t) + defer rulesClose() + setupPMAPIRules(rules) + + testTransferFrom(t, rules, provider, []Message{ + {ID: "draft1", Body: getTestMsgBody("draft1"), Targets: []Mailbox{{ID: pmapi.DraftLabel}}}, + }) +} + +func TestPMAPIProviderTransferFromTo(t *testing.T) { + m := initMocks(t) + defer m.ctrl.Finish() + + setupPMAPIClientExpectationForExport(&m) + setupPMAPIClientExpectationForImport(&m) + + source, err := NewPMAPIProvider(m.clientManager, "user", "addressID") + r.NoError(t, err) + target, err := NewPMAPIProvider(m.clientManager, "user", "addressID") + r.NoError(t, err) + + rules, rulesClose := newTestRules(t) + defer rulesClose() + setupPMAPIRules(rules) + + testTransferFromTo(t, rules, source, target, 5*time.Second) +} + +func setupPMAPIRules(rules transferRules) { + _ = rules.setRule(Mailbox{ID: pmapi.InboxLabel}, []Mailbox{{ID: pmapi.InboxLabel}}, 0, 0) +} + +func setupPMAPIClientExpectationForExport(m *mocks) { + m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes() + m.pmapiClient.EXPECT().ListLabels().Return([]*pmapi.Label{ + {ID: "label1", Name: "Foo", Color: "blue", Exclusive: 0, Order: 2}, + {ID: "label2", Name: "Bar", Color: "green", Exclusive: 0, Order: 1}, + {ID: "folder1", Name: "One", Color: "red", Exclusive: 1, Order: 1}, + {ID: "folder2", Name: "Two", Color: "orange", Exclusive: 1, Order: 2}, + }, nil).AnyTimes() + m.pmapiClient.EXPECT().CountMessages(gomock.Any()).Return([]*pmapi.MessagesCount{ + {LabelID: "label1", Total: 10}, + {LabelID: "label2", Total: 0}, + {LabelID: "folder1", Total: 20}, + }, nil).AnyTimes() + m.pmapiClient.EXPECT().ListMessages(gomock.Any()).Return([]*pmapi.Message{ + {ID: "msg1"}, + {ID: "msg2"}, + }, 2, nil).AnyTimes() + m.pmapiClient.EXPECT().GetMessage(gomock.Any()).DoAndReturn(func(msgID string) (*pmapi.Message, error) { + return &pmapi.Message{ + ID: msgID, + Body: string(getTestMsgBody(msgID)), + MIMEType: pmapi.ContentTypeMultipartMixed, + }, nil + }).AnyTimes() +} + +func setupPMAPIClientExpectationForImport(m *mocks) { + m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes() + m.pmapiClient.EXPECT().Import(gomock.Any()).DoAndReturn(func(requests []*pmapi.ImportMsgReq) ([]*pmapi.ImportMsgRes, error) { + r.Equal(m.t, 1, len(requests)) + + request := requests[0] + for _, msgID := range []string{"msg1", "msg2"} { + if bytes.Contains(request.Body, []byte(msgID)) { + return []*pmapi.ImportMsgRes{{MessageID: msgID, Error: nil}}, nil + } + } + r.Fail(m.t, "No message found") + return nil, nil + }).Times(2) +} + +func setupPMAPIClientExpectationForImportDraft(m *mocks) { + m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes() + m.pmapiClient.EXPECT().CreateDraft(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(msg *pmapi.Message, parentID string, action int) (*pmapi.Message, error) { + r.Equal(m.t, msg.Subject, "draft1") + msg.ID = "draft1" + return msg, nil + }) +} diff --git a/internal/transfer/provider_pmapi_utils.go b/internal/transfer/provider_pmapi_utils.go new file mode 100644 index 00000000..65bf8817 --- /dev/null +++ b/internal/transfer/provider_pmapi_utils.go @@ -0,0 +1,109 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "io" + "time" + + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/pkg/errors" +) + +const ( + pmapiRetries = 10 + pmapiReconnectTimeout = 30 * time.Minute + pmapiReconnectSleep = time.Minute +) + +func (p *PMAPIProvider) ensureConnection(callback func() error) error { + var callErr error + for i := 1; i <= pmapiRetries; i++ { + callErr = callback() + if callErr == nil { + return nil + } + + log.WithField("attempt", i).WithError(callErr).Warning("Call failed, trying reconnect") + err := p.tryReconnect() + if err != nil { + return err + } + } + return errors.Wrap(callErr, "too many retries") +} + +func (p *PMAPIProvider) tryReconnect() error { + start := time.Now() + var previousErr error + for { + if time.Since(start) > pmapiReconnectTimeout { + return previousErr + } + + err := p.clientManager.CheckConnection() + if err != nil { + time.Sleep(pmapiReconnectSleep) + previousErr = err + continue + } + + break + } + return nil +} + +func (p *PMAPIProvider) listMessages(filter *pmapi.MessagesFilter) (messages []*pmapi.Message, count int, err error) { + err = p.ensureConnection(func() error { + messages, count, err = p.client().ListMessages(filter) + return err + }) + return +} + +func (p *PMAPIProvider) getMessage(msgID string) (message *pmapi.Message, err error) { + err = p.ensureConnection(func() error { + message, err = p.client().GetMessage(msgID) + return err + }) + return +} + +func (p *PMAPIProvider) importRequest(req []*pmapi.ImportMsgReq) (res []*pmapi.ImportMsgRes, err error) { + err = p.ensureConnection(func() error { + res, err = p.client().Import(req) + return err + }) + return +} + +func (p *PMAPIProvider) createDraft(message *pmapi.Message, parent string, action int) (draft *pmapi.Message, err error) { + err = p.ensureConnection(func() error { + draft, err = p.client().CreateDraft(message, parent, action) + return err + }) + return +} + +func (p *PMAPIProvider) createAttachment(att *pmapi.Attachment, r io.Reader, sig io.Reader) (created *pmapi.Attachment, err error) { + err = p.ensureConnection(func() error { + created, err = p.client().CreateAttachment(att, r, sig) + return err + }) + return +} diff --git a/internal/transfer/provider_test.go b/internal/transfer/provider_test.go new file mode 100644 index 00000000..1b58a14d --- /dev/null +++ b/internal/transfer/provider_test.go @@ -0,0 +1,111 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "fmt" + "testing" + "time" + + a "github.com/stretchr/testify/assert" + r "github.com/stretchr/testify/require" +) + +func getTestMsgBody(subject string) []byte { + return []byte(fmt.Sprintf(`Subject: %s +From: Bridge Test +To: Bridge Test +Content-Type: multipart/mixed; boundary=c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a + +--c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a +Content-Disposition: inline +Content-Transfer-Encoding: 7bit +Content-Type: text/plain; charset=utf-8 + +hello + +--c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a-- +`, subject)) +} + +func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, expectedMessageIDs []string) { + progress := newProgress(log, nil) + drainProgressUpdateChannel(&progress) + + ch := make(chan Message) + go func() { + provider.TransferTo(rules, &progress, ch) + close(ch) + }() + + gotMessageIDs := []string{} + for msg := range ch { + gotMessageIDs = append(gotMessageIDs, msg.ID) + } + r.ElementsMatch(t, expectedMessageIDs, gotMessageIDs) + + r.Empty(t, progress.GetFailedMessages()) +} + +func testTransferFrom(t *testing.T, rules transferRules, provider TargetProvider, messages []Message) { + progress := newProgress(log, nil) + drainProgressUpdateChannel(&progress) + + ch := make(chan Message) + go func() { + for _, message := range messages { + progress.addMessage(message.ID, nil) + progress.messageExported(message.ID, []byte(""), nil) + ch <- message + } + close(ch) + }() + + go func() { + provider.TransferFrom(rules, &progress, ch) + progress.finish() + }() + + maxWait := time.Duration(len(messages)) * time.Second + a.Eventually(t, func() bool { + return progress.updateCh == nil + }, maxWait, 10*time.Millisecond, "Waiting for imported messages timed out") + + r.Empty(t, progress.GetFailedMessages()) +} + +func testTransferFromTo(t *testing.T, rules transferRules, source SourceProvider, target TargetProvider, maxWait time.Duration) { + progress := newProgress(log, nil) + drainProgressUpdateChannel(&progress) + + ch := make(chan Message) + go func() { + source.TransferTo(rules, &progress, ch) + close(ch) + }() + go func() { + target.TransferFrom(rules, &progress, ch) + progress.finish() + }() + + a.Eventually(t, func() bool { + return progress.updateCh == nil + }, maxWait, 10*time.Millisecond, "Waiting for export and import timed out") + + r.Empty(t, progress.GetFailedMessages()) +} diff --git a/internal/transfer/report.go b/internal/transfer/report.go new file mode 100644 index 00000000..a09d9b38 --- /dev/null +++ b/internal/transfer/report.go @@ -0,0 +1,145 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/pkg/errors" +) + +// fileReport is struct which can write and read message details. +// File report includes private information. +type fileReport struct { + path string +} + +func openLastFileReport(reportsPath, importID string) (*fileReport, error) { //nolint[deadcode] + allLogFileNames, err := getFilePathsWithSuffix(reportsPath, ".log") + if err != nil { + return nil, err + } + + reportFileNames := []string{} + for _, fileName := range allLogFileNames { + if strings.HasPrefix(fileName, fmt.Sprintf("import_%s_", importID)) { + reportFileNames = append(reportFileNames, fileName) + } + } + if len(reportFileNames) == 0 { + return nil, errors.New("no report found") + } + + sort.Strings(reportFileNames) + reportFileName := reportFileNames[len(reportFileNames)-1] + path := filepath.Join(reportsPath, reportFileName) + return &fileReport{ + path: path, + }, nil +} + +func newFileReport(reportsPath, importID string) *fileReport { + fileName := fmt.Sprintf("import_%s_%d.log", importID, time.Now().Unix()) + path := filepath.Join(reportsPath, fileName) + + return &fileReport{ + path: path, + } +} + +func (r *fileReport) writeMessageStatus(messageStatus *MessageStatus) { + f, err := os.OpenFile(r.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + log.WithError(err).Error("Failed to open report file") + } + defer f.Close() //nolint[errcheck] + + messageReport := newMessageReportFromMessageStatus(messageStatus, true) + data, err := json.Marshal(messageReport) + if err != nil { + log.WithError(err).Error("Failed to marshall message details") + } + data = append(data, '\n') + + if _, err = f.Write(data); err != nil { + log.WithError(err).Error("Failed to write to report file") + } +} + +// bugReport is struct which can create report for bug reporting. +// Bug report does NOT include private information. +type bugReport struct { + data bytes.Buffer +} + +func (r *bugReport) writeMessageStatus(messageStatus *MessageStatus) { + messageReport := newMessageReportFromMessageStatus(messageStatus, false) + data, err := json.Marshal(messageReport) + if err != nil { + log.WithError(err).Error("Failed to marshall message details") + } + _, _ = r.data.Write(data) + _, _ = r.data.Write([]byte("\n")) +} + +func (r *bugReport) getData() []byte { + return r.data.Bytes() +} + +// messageReport is struct which holds data used by `fileReport` and `bugReport`. +type messageReport struct { + EventTime int64 + SourceID string + TargetID string + BodyHash string + SourceMailbox string + TargetMailboxes []string + Error string + + // Private information for user. + Subject string + From string + Time string +} + +func newMessageReportFromMessageStatus(messageStatus *MessageStatus, includePrivateInfo bool) messageReport { + md := messageReport{ + EventTime: messageStatus.eventTime.Unix(), + SourceID: messageStatus.SourceID, + TargetID: messageStatus.targetID, + BodyHash: messageStatus.bodyHash, + SourceMailbox: messageStatus.rule.SourceMailbox.Name, + TargetMailboxes: messageStatus.rule.TargetMailboxNames(), + Error: messageStatus.GetErrorMessage(), + } + + if includePrivateInfo { + md.Subject = messageStatus.Subject + md.From = messageStatus.From + md.Time = messageStatus.Time.Format(time.RFC1123Z) + } + + return md +} diff --git a/internal/transfer/rules.go b/internal/transfer/rules.go new file mode 100644 index 00000000..2bf03915 --- /dev/null +++ b/internal/transfer/rules.go @@ -0,0 +1,290 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/pkg/errors" +) + +// transferRules maintains import rules, e.g. to which target mailbox should be +// source mailbox imported or what time spans. +type transferRules struct { + filePath string + + // rules is map with key as hash of source mailbox to its rule. + // Every source mailbox should have rule, at least disabled one. + rules map[string]*Rule + + // globalMailbox is applied to every message in the import phase. + // E.g., every message will be imported into this mailbox. + globalMailbox *Mailbox + + // skipEncryptedMessages determines whether message which cannot + // be decrypted should be exported or skipped. + skipEncryptedMessages bool +} + +// loadRules loads rules from `rulesPath` based on `ruleID`. +func loadRules(rulesPath, ruleID string) transferRules { + fileName := fmt.Sprintf("rules_%s.json", ruleID) + filePath := filepath.Join(rulesPath, fileName) + + var rules map[string]*Rule + f, err := os.Open(filePath) //nolint[gosec] + if err != nil { + log.WithError(err).Debug("Problem to read rules") + } else { + defer f.Close() //nolint[errcheck] + if err := json.NewDecoder(f).Decode(&rules); err != nil { + log.WithError(err).Warn("Problem to umarshal rules") + } + } + if rules == nil { + rules = map[string]*Rule{} + } + + return transferRules{ + filePath: filePath, + rules: rules, + } +} + +func (r *transferRules) setSkipEncryptedMessages(skip bool) { + r.skipEncryptedMessages = skip +} + +func (r *transferRules) setGlobalMailbox(mailbox *Mailbox) { + r.globalMailbox = mailbox +} + +func (r *transferRules) setGlobalTimeLimit(fromTime, toTime int64) { + for _, rule := range r.rules { + if !rule.HasTimeLimit() { + rule.FromTime = fromTime + rule.ToTime = toTime + } + } +} + +func (r *transferRules) getRuleBySourceMailboxName(name string) (*Rule, error) { + for _, rule := range r.rules { + if rule.SourceMailbox.Name == name { + return rule, nil + } + } + return nil, fmt.Errorf("no rule for mailbox %s", name) +} + +func (r *transferRules) iterateActiveRules() chan *Rule { + ch := make(chan *Rule) + go func() { + for _, rule := range r.rules { + if rule.Active { + ch <- rule + } + } + close(ch) + }() + return ch +} + +// setDefaultRules iterates `sourceMailboxes` and sets missing rules with +// matching mailboxes from `targetMailboxes`. In case no matching mailbox +// is found, `defaultCallback` with a source mailbox as a parameter is used. +func (r *transferRules) setDefaultRules(sourceMailboxes []Mailbox, targetMailboxes []Mailbox, defaultCallback func(Mailbox) []Mailbox) { + for _, sourceMailbox := range sourceMailboxes { + h := sourceMailbox.Hash() + if _, ok := r.rules[h]; ok { + continue + } + + targetMailboxes := sourceMailbox.findMatchingMailboxes(targetMailboxes) + if len(targetMailboxes) == 0 { + targetMailboxes = defaultCallback(sourceMailbox) + } + + active := true + if len(targetMailboxes) == 0 { + active = false + } + + // For both import to or export from ProtonMail, spam and draft + // mailboxes are by default deactivated. + for _, mailbox := range append([]Mailbox{sourceMailbox}, targetMailboxes...) { + if mailbox.ID == pmapi.SpamLabel || mailbox.ID == pmapi.DraftLabel || mailbox.ID == pmapi.TrashLabel { + active = false + break + } + } + + r.rules[h] = &Rule{ + Active: active, + SourceMailbox: sourceMailbox, + TargetMailboxes: targetMailboxes, + } + } + + for _, rule := range r.rules { + if !rule.Active { + continue + } + found := false + for _, sourceMailbox := range sourceMailboxes { + if sourceMailbox.Name == rule.SourceMailbox.Name { + found = true + } + } + if !found { + rule.Active = false + } + } + + r.save() +} + +// setRule sets messages from `sourceMailbox` between `fromData` and `toDate` +// (if used) to be imported to all `targetMailboxes`. +func (r *transferRules) setRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error { + numberOfExclusiveMailboxes := 0 + for _, mailbox := range targetMailboxes { + if mailbox.IsExclusive { + numberOfExclusiveMailboxes++ + } + } + if numberOfExclusiveMailboxes > 1 { + return errors.New("rule can have only one exclusive target mailbox") + } + + h := sourceMailbox.Hash() + r.rules[h] = &Rule{ + Active: true, + SourceMailbox: sourceMailbox, + TargetMailboxes: targetMailboxes, + FromTime: fromTime, + ToTime: toTime, + } + r.save() + return nil +} + +// unsetRule unsets messages from `sourceMailbox` to be exported. +func (r *transferRules) unsetRule(sourceMailbox Mailbox) { + h := sourceMailbox.Hash() + if rule, ok := r.rules[h]; ok { + rule.Active = false + } else { + r.rules[h] = &Rule{ + Active: false, + SourceMailbox: sourceMailbox, + } + } + r.save() +} + +// getRule returns rule for `sourceMailbox` or nil if it does not exist. +func (r *transferRules) getRule(sourceMailbox Mailbox) *Rule { + h := sourceMailbox.Hash() + return r.rules[h] +} + +// getRules returns all set rules. +func (r *transferRules) getRules() []*Rule { + rules := []*Rule{} + for _, rule := range r.rules { + rules = append(rules, rule) + } + return rules +} + +// reset wipes our all rules. +func (r *transferRules) reset() { + r.rules = map[string]*Rule{} + r.save() +} + +// save saves rules to file. +func (r *transferRules) save() { + f, err := os.Create(r.filePath) + if err != nil { + log.WithError(err).Warn("Problem to write rules") + return + } + defer f.Close() //nolint[errcheck] + + if err := json.NewEncoder(f).Encode(r.rules); err != nil { + log.WithError(err).Warn("Problem to marshal rules") + } +} + +// Rule is data holder of rule for one source mailbox used by `transferRules`. +type Rule struct { + Active bool `json:"active"` + SourceMailbox Mailbox `json:"source"` + TargetMailboxes []Mailbox `json:"targets"` + FromTime int64 `json:"from"` + ToTime int64 `json:"to"` +} + +// String returns textual representation for log purposes. +func (r *Rule) String() string { + return fmt.Sprintf( + "%s -> %s (%d - %d)", + r.SourceMailbox.Name, + strings.Join(r.TargetMailboxNames(), ", "), + r.FromTime, + r.ToTime, + ) +} + +func (r *Rule) isTimeInRange(t int64) bool { + if !r.HasTimeLimit() { + return true + } + return r.FromTime <= t && t <= r.ToTime +} + +// HasTimeLimit returns whether rule defines time limit. +func (r *Rule) HasTimeLimit() bool { + return r.FromTime != 0 || r.ToTime != 0 +} + +// FromDate returns time struct based on `FromTime`. +func (r *Rule) FromDate() time.Time { + return time.Unix(r.FromTime, 0) +} + +// ToDate returns time struct based on `ToTime`. +func (r *Rule) ToDate() time.Time { + return time.Unix(r.ToTime, 0) +} + +// TargetMailboxNames returns array of target mailbox names. +func (r *Rule) TargetMailboxNames() (names []string) { + for _, mailbox := range r.TargetMailboxes { + names = append(names, mailbox.Name) + } + return +} diff --git a/internal/transfer/rules_test.go b/internal/transfer/rules_test.go new file mode 100644 index 00000000..f887a8ac --- /dev/null +++ b/internal/transfer/rules_test.go @@ -0,0 +1,210 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + r "github.com/stretchr/testify/require" +) + +func newTestRules(t *testing.T) (transferRules, func()) { + path, err := ioutil.TempDir("", "rules") + r.NoError(t, err) + + ruleID := "rule" + rules := loadRules(path, ruleID) + return rules, func() { + _ = os.RemoveAll(path) + } +} + +func TestLoadRules(t *testing.T) { + path, err := ioutil.TempDir("", "rules") + r.NoError(t, err) + defer os.RemoveAll(path) //nolint[errcheck] + + ruleID := "rule" + rules := loadRules(path, ruleID) + + mailboxA := Mailbox{ID: "1", Name: "One", Color: "orange", IsExclusive: true} + mailboxB := Mailbox{ID: "2", Name: "Two", Color: "", IsExclusive: true} + mailboxC := Mailbox{ID: "3", Name: "Three", Color: "", IsExclusive: false} + + r.NoError(t, rules.setRule(mailboxA, []Mailbox{mailboxB, mailboxC}, 0, 0)) + r.NoError(t, rules.setRule(mailboxB, []Mailbox{mailboxB}, 10, 20)) + r.NoError(t, rules.setRule(mailboxC, []Mailbox{}, 0, 30)) + + rules2 := loadRules(path, ruleID) + r.Equal(t, map[string]*Rule{ + mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB, mailboxC}, FromTime: 0, ToTime: 0}, + mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 10, ToTime: 20}, + mailboxC.Hash(): {Active: true, SourceMailbox: mailboxC, TargetMailboxes: []Mailbox{}, FromTime: 0, ToTime: 30}, + }, rules2.rules) + + rules2.unsetRule(mailboxA) + rules2.unsetRule(mailboxC) + + rules3 := loadRules(path, ruleID) + r.Equal(t, map[string]*Rule{ + mailboxA.Hash(): {Active: false, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB, mailboxC}, FromTime: 0, ToTime: 0}, + mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 10, ToTime: 20}, + mailboxC.Hash(): {Active: false, SourceMailbox: mailboxC, TargetMailboxes: []Mailbox{}, FromTime: 0, ToTime: 30}, + }, rules3.rules) +} + +func TestSetGlobalTimeLimit(t *testing.T) { + path, err := ioutil.TempDir("", "rules") + r.NoError(t, err) + defer os.RemoveAll(path) //nolint[errcheck] + + rules := loadRules(path, "rule") + + mailboxA := Mailbox{Name: "One"} + mailboxB := Mailbox{Name: "Two"} + + r.NoError(t, rules.setRule(mailboxA, []Mailbox{}, 10, 20)) + r.NoError(t, rules.setRule(mailboxB, []Mailbox{}, 0, 0)) + + rules.setGlobalTimeLimit(30, 40) + + r.Equal(t, map[string]*Rule{ + mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{}, FromTime: 10, ToTime: 20}, + mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{}, FromTime: 30, ToTime: 40}, + }, rules.rules) +} + +func TestSetDefaultRules(t *testing.T) { + path, err := ioutil.TempDir("", "rules") + r.NoError(t, err) + defer os.RemoveAll(path) //nolint[errcheck] + + rules := loadRules(path, "rule") + + mailbox1 := Mailbox{Name: "One"} // Set manually, default will not override it. + mailbox2 := Mailbox{Name: "Two"} // Matched by `targetMailboxes`. + mailbox3 := Mailbox{Name: "Three"} // Matched by `defaultCallback`, not included in `targetMailboxes`. + mailbox4 := Mailbox{Name: "Four"} // Matched by nothing, will not be active. + mailbox5 := Mailbox{Name: "Spam", ID: pmapi.SpamLabel} // Spam is inactive by default (ID found in source). + mailbox6a := Mailbox{Name: "Draft"} // Draft is inactive by default (ID found in target, mailbox6b). + mailbox6b := Mailbox{Name: "Draft", ID: pmapi.DraftLabel} + + sourceMailboxes := []Mailbox{mailbox1, mailbox2, mailbox3, mailbox4, mailbox5, mailbox6a} + targetMailboxes := []Mailbox{mailbox1, mailbox2, mailbox6b} + + r.NoError(t, rules.setRule(mailbox1, []Mailbox{mailbox3}, 0, 0)) + + defaultCallback := func(mailbox Mailbox) []Mailbox { + if mailbox.Name == "Three" { + return []Mailbox{mailbox3} + } + return []Mailbox{} + } + + rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback) + + r.Equal(t, map[string]*Rule{ + mailbox1.Hash(): {Active: true, SourceMailbox: mailbox1, TargetMailboxes: []Mailbox{mailbox3}}, + mailbox2.Hash(): {Active: true, SourceMailbox: mailbox2, TargetMailboxes: []Mailbox{mailbox2}}, + mailbox3.Hash(): {Active: true, SourceMailbox: mailbox3, TargetMailboxes: []Mailbox{mailbox3}}, + mailbox4.Hash(): {Active: false, SourceMailbox: mailbox4, TargetMailboxes: []Mailbox{}}, + mailbox5.Hash(): {Active: false, SourceMailbox: mailbox5, TargetMailboxes: []Mailbox{}}, + mailbox6a.Hash(): {Active: false, SourceMailbox: mailbox6a, TargetMailboxes: []Mailbox{mailbox6b}}, + }, rules.rules) +} + +func TestSetDefaultRulesDeactivateMissing(t *testing.T) { + path, err := ioutil.TempDir("", "rules") + r.NoError(t, err) + defer os.RemoveAll(path) //nolint[errcheck] + + rules := loadRules(path, "rule") + + mailboxA := Mailbox{ID: "1", Name: "One", Color: "", IsExclusive: true} + mailboxB := Mailbox{ID: "2", Name: "Two", Color: "", IsExclusive: true} + + r.NoError(t, rules.setRule(mailboxA, []Mailbox{mailboxB}, 0, 0)) + r.NoError(t, rules.setRule(mailboxB, []Mailbox{mailboxB}, 0, 0)) + + sourceMailboxes := []Mailbox{mailboxA} + targetMailboxes := []Mailbox{mailboxA, mailboxB} + defaultCallback := func(mailbox Mailbox) (mailboxes []Mailbox) { + return + } + rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback) + + r.Equal(t, map[string]*Rule{ + mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 0, ToTime: 0}, + mailboxB.Hash(): {Active: false, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 0, ToTime: 0}, + }, rules.rules) +} + +func TestIsTimeInRange(t *testing.T) { + tests := []struct { + rule Rule + time int64 + want bool + }{ + {generateTimeRule(0, 0), 0, true}, + {generateTimeRule(0, 0), 10, true}, + {generateTimeRule(0, 15), 10, true}, + {generateTimeRule(5, 15), 10, true}, + {generateTimeRule(0, 5), 10, false}, + {generateTimeRule(5, 7), 10, false}, + {generateTimeRule(15, 30), 10, false}, + {generateTimeRule(15, 0), 10, false}, + } + for _, tc := range tests { + tc := tc + t.Run(fmt.Sprintf("%v / %d", tc.rule, tc.time), func(t *testing.T) { + got := tc.rule.isTimeInRange(tc.time) + r.Equal(t, tc.want, got) + }) + } +} + +func TestHasTimeLimit(t *testing.T) { + tests := []struct { + rule Rule + want bool + }{ + {generateTimeRule(0, 0), false}, + {generateTimeRule(0, 1), true}, + {generateTimeRule(1, 2), true}, + {generateTimeRule(1, 0), true}, + } + for _, tc := range tests { + tc := tc + t.Run(fmt.Sprintf("%v", tc.rule), func(t *testing.T) { + r.Equal(t, tc.want, tc.rule.HasTimeLimit()) + }) + } +} + +func generateTimeRule(from, to int64) Rule { + return Rule{ + SourceMailbox: Mailbox{}, + TargetMailboxes: []Mailbox{}, + FromTime: from, + ToTime: to, + } +} diff --git a/internal/transfer/testdata/eml/Foo/msg.eml b/internal/transfer/testdata/eml/Foo/msg.eml new file mode 100644 index 00000000..3b342260 --- /dev/null +++ b/internal/transfer/testdata/eml/Foo/msg.eml @@ -0,0 +1,4 @@ +From: Bridge Test +To: Bridge Test + +hello diff --git a/internal/transfer/testdata/eml/Inbox/msg.eml b/internal/transfer/testdata/eml/Inbox/msg.eml new file mode 100644 index 00000000..3b342260 --- /dev/null +++ b/internal/transfer/testdata/eml/Inbox/msg.eml @@ -0,0 +1,4 @@ +From: Bridge Test +To: Bridge Test + +hello diff --git a/internal/transfer/testdata/emlmbox/Foo/msg.eml b/internal/transfer/testdata/emlmbox/Foo/msg.eml new file mode 100644 index 00000000..3b342260 --- /dev/null +++ b/internal/transfer/testdata/emlmbox/Foo/msg.eml @@ -0,0 +1,4 @@ +From: Bridge Test +To: Bridge Test + +hello diff --git a/internal/transfer/testdata/emlmbox/Inbox.mbox b/internal/transfer/testdata/emlmbox/Inbox.mbox new file mode 100644 index 00000000..25ad1c5b --- /dev/null +++ b/internal/transfer/testdata/emlmbox/Inbox.mbox @@ -0,0 +1,5 @@ +From - Mon May 4 16:40:31 2020 +From: Bridge Test +To: Bridge Test + +hello diff --git a/internal/transfer/testdata/keyring_userKey b/internal/transfer/testdata/keyring_userKey new file mode 100644 index 00000000..976d2be2 --- /dev/null +++ b/internal/transfer/testdata/keyring_userKey @@ -0,0 +1,62 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: OpenPGP.js v4.4.5 +Comment: testpassphrase + +xcLYBFzGzhEBCADBxfqTFMqfQzT77A5tuuhPFwPq8dfC2evs8u1OvTqFbztY +5FOuSxzduyeDqQ1Fx6dKEOKgcYE8t1Uh4VSS7z6bTdY8j9yrL81kCVB46sE1 +OzStzyx/5l7OdH/pM4F+aKslnLvqlw0UeJr+UNizVtOCEUaNfVjPK3cc1ocx +v+36K4RnnyfEtjUW9gDZbhgaF02G5ILHmWmbgM7I+77gCd2wI0EdY9s/JZQ+ +VmkMFqoMdY9PyBchoOIPUkkGQi1SaF4IEzMaAUSbnCYkHHY/SbfDTcR46VGq +cXlkB1rq5xskaUQ9r+giCC/K4pc7bBkI1lQ7ADVuWvdrWnWapK0FO6CfABEB +AAEAB/0YPhPJ0phA/EWviN+16bmGVOZNaVapjt2zMMybWmrtEQv3OeWgO3nP +4cohRi/zaCBCphcm+dxbLhftW7AFi/9PVcR09436MB+oTCQFugpUWw+4TmA5 +BidxTpDxf4X2vH3rquQLBufWL6U7JlPeKAGL1xZ2aCq0DIeOk5D+xTjZizV2 +GIyQRVCLWb+LfDmvvcp3Y94X60KXdBAMuS1ZMKcY3Sl8VAXNB4KQsC/kByzf +6FCB097XZRYV7lvJJQ7+6Wisb3yVi8sEQx2sFm5fAp+0qi3a6zRTEp49r6Hr +gyWViH5zOOpA7DcNwx1Bwhi7GG0tak6EUnnKUNLfOupglcphBADmpXCgT4nc +uSBYTiZSVcB/ICCkTxVsHL1WcXtPK2Ikzussx2n9kb0rapvuC0YLipX9lUkQ +fyeC3jQJeCyN79AkDGkOfWaESueT2hM0Po+RwDgMibKn6yJ1zebz4Lc2J3C9 +oVFcAnql+9KyGsAPn03fyQzDnvhNnJvHJi4Hx8AWoQQA1xLoXeVBjRi0IjjU +E6Mqaq5RLEog4kXRp86VSSEGHBwyIYnDiM//gjseo/CXuVyHwL7UXitp8s1B +D1uE3APrhqUS66fD5pkF+z+RcSqiIv7I76NJ24Cdg38L6seGSjOHrq7/dEeG +K6WqfQUCEjta3yNSg7pXb2wn2WZqKIK+rz8EALZRuMXeql/FtO3Cjb0sv7oT +9dLP4cn1bskGRJ+Vok9lfCERbfXGccoAk3V+qSfpHgKxsebkRbUhf+trOGnw +tW+kBWo/5hYGQuN+A9JogSJViT+nuZyE+x1/rKswDFmlMSdf2GIDARWIV0gc +b1yOEwUmNBSthPcnFXvBr4BG3XTtNPTNLSJhcm9uMjEtM0BzYWRlbWJlLm9y +ZyIgPGFyb24yMS0zQHNhZGVtYmUub3JnPsLAdQQQAQgAHwUCXMbOEQYLCQcI +AwIEFQgKAgMWAgECGQECGwMCHgEACgkQZ/B4v2b2xB6XUgf/dHGRHimyMR78 +QYbEm2cuaEvOtq4a+J6Zv3P4VOWAbvkGWS9LDKSvVi60vq4oYOmF54HgPzur +nA4OtZDf0HKwQK45VZ7CYD693o70jkKPrAAJG3yTsbesfiS7RbFyGKzKJ7EL +nsUIJkfgm/SlKmXU/u8MOBO5Wg7/TcsS33sRWHl90j+9jbhqdl92R+vY/CwC +ieFkQA7/TDv1u+NAalH+Lpkd8AIuEcki+TAogZ7oi/SnofwnoB7BxRm+mIkp +ZZhIDSCaPOzLG8CSZ81d3HVHhqbf8dh0DFKFoUYyKdbOqIkNWWASf+c/ZEme +IWcekY8hqwf/raZ56tGM/bRwYPcotMfC1wRcxs4RAQgAsMb5/ELWmrfPy3ba +5qif+RXhGSbjitATNgHpoPUHrfTC7cn4JWHqehoXLAQpFAoKd+O/ZNpZozK9 +ilpqGUx05yMw06jNQEhYIbgIF4wzPpz02Lp6YeMwdF5LF+Rw83PHdHrA/wRV +/QjL04+kZnN+G5HmzMlhFY+oZSpL+Gp1bTXgtAVDkhCnMB5tP2VwULMGyJ+X +vRYxwTK2CrLjIVZv5n1VYY+caCowU6j/XFqvlCJj+G5oV+UhFOWffaMRXhOh +a64RrhqT1Np7wCLvLMP2wpys9xlMcLQJLqDNxqOTp504V7dm67ncC0fKUsT4 +m4oTktnxKPd6MU+4VYveaLCquwARAQABAAf4u9s7gpGErs1USxmDO9TlyGZK +aBlri8nMf3s+hOJCOo3cRaRHJBfdY6pu/baG6H6JTsWzeY4MHwr6N+dhVIEh +FPMa9EZAjagyc4GugxWGiMVTfU+2AEfdrdynhQKMgXSctnnNCdkRuX0nwqb3 +nlupm1hsz2ze4+Wg0BKSLS0FQdoUbITdJUR69OHr4dNJVHWYI0JSBx4SdhV3 +y9163dDvmc+lW9AEaD53vyZWfzCHZxsR/gI32VmT0z5gn1t8w9AOdXo2lA1H +bf7wh4/qCyujGu64ToZtiEny/GCyM6PofLtiZuJNLw3s/y+B2tKv22aTJ760 ++Gib1xB9WcWjKyrxBADoeCyq+nHGrl0CwOkmjanlFymgo7mnBOXuiFOvGrKk +M1meMU1TI4TEBWkVnDVMcSejgjAf/bX1dtouba1tMAMu7DlaV/0EwbSADRel +RSqEbIzIOys+y9TY/BMI/uCKNyEKHvu1KUXADb+CBpdBpCfMBWDANFlo9xLz +Ajcmu2dyawQAwquwC0VXQcvzfs+Hd5au5XvHdm1KidOiAdu6PH1SrOgenIN4 +lkEjHrJD9jmloO2/GVcxDBB2pmf0B4HEg7DuY9LXBrksP5eSbbRc5+UH1HUv +u82AqQnfNKTd/jae+lLwaOS++ohtwMkkD6W0LdWnHPjyyXg4Oi9zPID3asRu +3PED/3CYyjl6S8GTMY4FNH7Yxu9+NV2xpKE92Hf8K/hnYlmSSVKDCEeOJtLt +BkkcSqY6liCNSMmJdVyAF2GrR+zmDac7UQRssf57oOWtSsGozt0aqJXuspMT +6aB+P1UhZ8Ly9rWZNiJ0jwyfnQNOLCYDaqjFmiSpqrNnJ2Q1Xge3+k80P9DC +wF8EGAEIAAkFAlzGzhECGwwACgkQZ/B4v2b2xB5wlwgAjZA1zdv5irFjyWVo +4/itONtyO1NbdpyYpcct7vD0oV+a4wahQP0J3Kk1GhZ5tvAoZF/jakQQOM5o +GjUYpXAGnr09Mv9EiQ2pDwXc2yq0WfXnGxNrpzOqdtV+IqY9NYkl55Tme7x+ +WRvrkPSUeUsyEGvxwR1stdv8eg9jUmxdl8Io3PYoFJJlrM/6aXeC1r3KOj7q +XAnR0XHJ+QBSNKCWLlQv5hui9BKfcLiVKFK/dNhs82nRyhPr4sWFw6MTqdAK +4zkn7l0jmy6Evi1AiiGPiHPnxeNErnofOIEh4REQj00deZADHrixTLtx2FuR +uaSC3IcBmBsj1fNb4eYXElILjQ== +=fMOl +-----END PGP PRIVATE KEY BLOCK----- \ No newline at end of file diff --git a/internal/transfer/testdata/mbox/Foo.mbox b/internal/transfer/testdata/mbox/Foo.mbox new file mode 100644 index 00000000..25ad1c5b --- /dev/null +++ b/internal/transfer/testdata/mbox/Foo.mbox @@ -0,0 +1,5 @@ +From - Mon May 4 16:40:31 2020 +From: Bridge Test +To: Bridge Test + +hello diff --git a/internal/transfer/testdata/mbox/Inbox.mbox b/internal/transfer/testdata/mbox/Inbox.mbox new file mode 100644 index 00000000..25ad1c5b --- /dev/null +++ b/internal/transfer/testdata/mbox/Inbox.mbox @@ -0,0 +1,5 @@ +From - Mon May 4 16:40:31 2020 +From: Bridge Test +To: Bridge Test + +hello diff --git a/internal/transfer/transfer.go b/internal/transfer/transfer.go new file mode 100644 index 00000000..bcd7290d --- /dev/null +++ b/internal/transfer/transfer.go @@ -0,0 +1,177 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +// Package transfer provides tools to export messages from one provider and +// import them to another provider. Provider can be EML, MBOX, IMAP or PMAPI. +package transfer + +import ( + "crypto/sha256" + "fmt" + + "github.com/sirupsen/logrus" +) + +var log = logrus.WithField("pkg", "transfer") //nolint[gochecknoglobals] + +// Transfer is facade on top of import rules, progress manager and source +// and target providers. This is the main object which should be used. +type Transfer struct { + panicHandler PanicHandler + id string + dir string + rules transferRules + source SourceProvider + target TargetProvider +} + +// New creates Transfer for specific source and target. Usage: +// +// source := transfer.NewEMLProvider(...) +// target := transfer.NewPMAPIProvider(...) +// transfer.New(source, target, ...) +func New(panicHandler PanicHandler, transferDir string, source SourceProvider, target TargetProvider) (*Transfer, error) { + transferID := fmt.Sprintf("%x", sha256.Sum256([]byte(source.ID()+"-"+target.ID()))) + rules := loadRules(transferDir, transferID) + transfer := &Transfer{ + panicHandler: panicHandler, + id: transferID, + dir: transferDir, + rules: rules, + source: source, + target: target, + } + if err := transfer.setDefaultRules(); err != nil { + return nil, err + } + return transfer, nil +} + +// SetDefaultRules sets missing rules for source mailboxes with matching +// target mailboxes. In case no matching mailbox is found, `defaultCallback` +// with a source mailbox as a parameter is used. +func (t *Transfer) setDefaultRules() error { + sourceMailboxes, err := t.SourceMailboxes() + if err != nil { + return err + } + + targetMailboxes, err := t.TargetMailboxes() + if err != nil { + return err + } + + defaultCallback := func(sourceMailbox Mailbox) []Mailbox { + return t.target.DefaultMailboxes(sourceMailbox) + } + + t.rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback) + return nil +} + +// SetSkipEncryptedMessages sets whether message which cannot be decrypted +// should be exported or skipped. +func (t *Transfer) SetSkipEncryptedMessages(skip bool) { + t.rules.setSkipEncryptedMessages(skip) +} + +// SetGlobalMailbox sets mailbox that is applied to every message in +// the import phase. +func (t *Transfer) SetGlobalMailbox(mailbox *Mailbox) { + t.rules.setGlobalMailbox(mailbox) +} + +// SetGlobalTimeLimit sets time limit that is applied to rules without any +// specified time limit. +func (t *Transfer) SetGlobalTimeLimit(fromTime, toTime int64) { + t.rules.setGlobalTimeLimit(fromTime, toTime) +} + +// SetRule sets sourceMailbox for transfer. +func (t *Transfer) SetRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error { + return t.rules.setRule(sourceMailbox, targetMailboxes, fromTime, toTime) +} + +// UnsetRule unsets sourceMailbox from transfer. +func (t *Transfer) UnsetRule(sourceMailbox Mailbox) { + t.rules.unsetRule(sourceMailbox) +} + +// ResetRules unsets all rules. +func (t *Transfer) ResetRules() { + t.rules.reset() +} + +// GetRule returns rule for given mailbox. +func (t *Transfer) GetRule(sourceMailbox Mailbox) *Rule { + return t.rules.getRule(sourceMailbox) +} + +// GetRules returns all set transfer rules. +func (t *Transfer) GetRules() []*Rule { + return t.rules.getRules() +} + +// SourceMailboxes returns mailboxes available at source side. +func (t *Transfer) SourceMailboxes() ([]Mailbox, error) { + return t.source.Mailboxes(false, true) +} + +// TargetMailboxes returns mailboxes available at target side. +func (t *Transfer) TargetMailboxes() ([]Mailbox, error) { + return t.target.Mailboxes(true, false) +} + +// CreateTargetMailbox creates mailbox in target provider. +func (t *Transfer) CreateTargetMailbox(mailbox Mailbox) (Mailbox, error) { + return t.target.CreateMailbox(mailbox) +} + +// ChangeTarget allows to change target. Ideally should not be used. +// Useful for situration after user changes mind where to export files and similar. +func (t *Transfer) ChangeTarget(target TargetProvider) { + t.target = target +} + +// Start starts the transfer from source to target. +func (t *Transfer) Start() *Progress { + log.Debug("Transfer started") + t.rules.save() + + log := log.WithField("id", t.id) + reportFile := newFileReport(t.dir, t.id) + progress := newProgress(log, reportFile) + + ch := make(chan Message) + + go func() { + defer t.panicHandler.HandlePanic() + + progress.start() + t.source.TransferTo(t.rules, &progress, ch) + close(ch) + }() + + go func() { + defer t.panicHandler.HandlePanic() + + t.target.TransferFrom(t.rules, &progress, ch) + progress.finish() + }() + + return &progress +} diff --git a/internal/transfer/transfer_test.go b/internal/transfer/transfer_test.go new file mode 100644 index 00000000..9a7c0bf7 --- /dev/null +++ b/internal/transfer/transfer_test.go @@ -0,0 +1,73 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/ProtonMail/gopenpgp/crypto" + transfermocks "github.com/ProtonMail/proton-bridge/internal/transfer/mocks" + pmapimocks "github.com/ProtonMail/proton-bridge/pkg/pmapi/mocks" + gomock "github.com/golang/mock/gomock" +) + +type mocks struct { + t *testing.T + + ctrl *gomock.Controller + panicHandler *transfermocks.MockPanicHandler + clientManager *transfermocks.MockClientManager + pmapiClient *pmapimocks.MockClient + + keyring *crypto.KeyRing +} + +func initMocks(t *testing.T) mocks { + mockCtrl := gomock.NewController(t) + + m := mocks{ + t: t, + + ctrl: mockCtrl, + panicHandler: transfermocks.NewMockPanicHandler(mockCtrl), + clientManager: transfermocks.NewMockClientManager(mockCtrl), + pmapiClient: pmapimocks.NewMockClient(mockCtrl), + keyring: newTestKeyring(), + } + + m.clientManager.EXPECT().GetClient("user").Return(m.pmapiClient).AnyTimes() + + return m +} + +func newTestKeyring() *crypto.KeyRing { + data, err := ioutil.ReadFile("testdata/keyring_userKey") + if err != nil { + panic(err) + } + userKey, err := crypto.ReadArmoredKeyRing(bytes.NewReader(data)) + if err != nil { + panic(err) + } + if err := userKey.Unlock([]byte("testpassphrase")); err != nil { + panic(err) + } + return userKey +} diff --git a/internal/transfer/types.go b/internal/transfer/types.go new file mode 100644 index 00000000..9b12db49 --- /dev/null +++ b/internal/transfer/types.go @@ -0,0 +1,31 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "github.com/ProtonMail/proton-bridge/pkg/pmapi" +) + +type PanicHandler interface { + HandlePanic() +} + +type ClientManager interface { + GetClient(userID string) pmapi.Client + CheckConnection() error +} diff --git a/internal/transfer/utils.go b/internal/transfer/utils.go new file mode 100644 index 00000000..17bd62c7 --- /dev/null +++ b/internal/transfer/utils.go @@ -0,0 +1,141 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "bufio" + "bytes" + "io/ioutil" + "net/mail" + "net/textproto" + "path/filepath" + "sort" + "strings" + + "github.com/pkg/errors" +) + +// getFolderNames collects all folder names under `root`. +// Folder names will be without a path. +func getFolderNames(root string) ([]string, error) { + return getFolderNamesWithFileSuffix(root, "") +} + +// getFolderNamesWithFileSuffix collects all folder names under `root`, which +// contains some file with a give `fileSuffix`. Names will be without a path. +func getFolderNamesWithFileSuffix(root, fileSuffix string) ([]string, error) { + folders := []string{} + + files, err := ioutil.ReadDir(root) + if err != nil { + return nil, err + } + + hasFileWithSuffix := fileSuffix == "" + for _, file := range files { + if file.IsDir() { + subfolders, err := getFolderNamesWithFileSuffix(filepath.Join(root, file.Name()), fileSuffix) + if err != nil { + return nil, err + } + for _, subfolder := range subfolders { + match := false + for _, folder := range folders { + if folder == subfolder { + match = true + break + } + } + if !match { + folders = append(folders, subfolder) + } + } + } else if fileSuffix == "" || strings.HasSuffix(file.Name(), fileSuffix) { + hasFileWithSuffix = true + } + } + + if hasFileWithSuffix { + folders = append(folders, filepath.Base(root)) + } + + sort.Strings(folders) + return folders, nil +} + +// getFilePathsWithSuffix collects all file names with `suffix` under `root`. +// File names will be with relative path based to `root`. +func getFilePathsWithSuffix(root, suffix string) ([]string, error) { + fileNames, err := getFilePathsWithSuffixInner("", root, suffix) + if err != nil { + return nil, err + } + sort.Strings(fileNames) + return fileNames, err +} + +func getFilePathsWithSuffixInner(prefix, root, suffix string) ([]string, error) { + fileNames := []string{} + + files, err := ioutil.ReadDir(root) + if err != nil { + return nil, err + } + + for _, file := range files { + if !file.IsDir() { + if strings.HasSuffix(file.Name(), suffix) { + fileNames = append(fileNames, filepath.Join(prefix, file.Name())) + } + } else { + subfolderFileNames, err := getFilePathsWithSuffixInner( + filepath.Join(prefix, file.Name()), + filepath.Join(root, file.Name()), + suffix, + ) + if err != nil { + return nil, err + } + fileNames = append(fileNames, subfolderFileNames...) + } + } + + return fileNames, nil +} + +// getMessageTime returns time of the message specified in the message header. +func getMessageTime(body []byte) (int64, error) { + mailHeader, err := getMessageHeader(body) + if err != nil { + return 0, err + } + if t, err := mailHeader.Date(); err == nil && !t.IsZero() { + return t.Unix(), nil + } + return 0, nil +} + +// getMessageHeader returns headers of the message body. +func getMessageHeader(body []byte) (mail.Header, error) { + tpr := textproto.NewReader(bufio.NewReader(bytes.NewBuffer(body))) + header, err := tpr.ReadMIMEHeader() + if err != nil { + return nil, errors.Wrap(err, "failed to read headers") + } + return mail.Header(header), nil +} diff --git a/internal/transfer/utils_test.go b/internal/transfer/utils_test.go new file mode 100644 index 00000000..fc2d83e0 --- /dev/null +++ b/internal/transfer/utils_test.go @@ -0,0 +1,190 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + r "github.com/stretchr/testify/require" +) + +func TestGetFolderNames(t *testing.T) { + root, clean := createTestingFolderStructure(t) + defer clean() + + tests := []struct { + suffix string + wantNames []string + }{ + { + "", + []string{ + "bar", + "baz", + filepath.Base(root), + "foo", + "qwerty", + "test", + }, + }, + { + ".eml", + []string{ + "bar", + "baz", + filepath.Base(root), + "foo", + }, + }, + { + ".txt", + []string{ + filepath.Base(root), + }, + }, + } + for _, tc := range tests { + tc := tc + t.Run(tc.suffix, func(t *testing.T) { + names, err := getFolderNamesWithFileSuffix(root, tc.suffix) + r.NoError(t, err) + r.Equal(t, tc.wantNames, names) + }) + } +} + +func TestGetFilePathsWithSuffix(t *testing.T) { + root, clean := createTestingFolderStructure(t) + defer clean() + + tests := []struct { + suffix string + wantPaths []string + }{ + { + ".eml", + []string{ + "foo/bar/baz/msg1.eml", + "foo/bar/baz/msg2.eml", + "foo/bar/baz/msg3.eml", + "foo/bar/msg4.eml", + "foo/bar/msg5.eml", + "foo/baz/msg6.eml", + "foo/msg7.eml", + "msg10.eml", + "test/foo/msg8.eml", + "test/foo/msg9.eml", + }, + }, + { + ".txt", + []string{ + "info.txt", + }, + }, + { + ".hello", + []string{}, + }, + } + for _, tc := range tests { + tc := tc + t.Run(tc.suffix, func(t *testing.T) { + paths, err := getFilePathsWithSuffix(root, tc.suffix) + r.NoError(t, err) + r.Equal(t, tc.wantPaths, paths) + }) + } +} + +func createTestingFolderStructure(t *testing.T) (string, func()) { + root, err := ioutil.TempDir("", "folderstructure") + r.NoError(t, err) + + for _, path := range []string{ + "foo/bar/baz", + "foo/baz", + "test/foo", + "qwerty", + } { + err = os.MkdirAll(filepath.Join(root, path), os.ModePerm) + r.NoError(t, err) + } + + for _, path := range []string{ + "foo/bar/baz/msg1.eml", + "foo/bar/baz/msg2.eml", + "foo/bar/baz/msg3.eml", + "foo/bar/msg4.eml", + "foo/bar/msg5.eml", + "foo/baz/msg6.eml", + "foo/msg7.eml", + "test/foo/msg8.eml", + "test/foo/msg9.eml", + "msg10.eml", + "info.txt", + } { + f, err := os.Create(filepath.Join(root, path)) + r.NoError(t, err) + err = f.Close() + r.NoError(t, err) + } + + return root, func() { + _ = os.RemoveAll(root) + } +} + +func TestGetMessageTime(t *testing.T) { + tests := []struct { + body string + wantTime int64 + wantErr string + }{ + {"", 0, "failed to read headers: EOF"}, + {"Subject: hello\n\n", 0, ""}, + {"Date: Thu, 23 Apr 2020 04:52:44 +0000\n\n", 1587617564, ""}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.body, func(t *testing.T) { + time, err := getMessageTime([]byte(tc.body)) + if tc.wantErr == "" { + r.NoError(t, err) + } else { + r.EqualError(t, err, tc.wantErr) + } + r.Equal(t, tc.wantTime, time) + }) + } +} + +func TestGetMessageHeader(t *testing.T) { + body := `Subject: Hello +From: user@example.com + +Body +` + header, err := getMessageHeader([]byte(body)) + r.NoError(t, err) + r.Equal(t, header.Get("subject"), "Hello") + r.Equal(t, header.Get("from"), "user@example.com") +} diff --git a/internal/users/user.go b/internal/users/user.go index 35e45982..159988b5 100644 --- a/internal/users/user.go +++ b/internal/users/user.go @@ -359,14 +359,17 @@ func (u *User) GetAddressID(address string) (id string, err error) { u.lock.RLock() defer u.lock.RUnlock() - address = strings.ToLower(address) - - if u.store == nil { - err = errors.New("store is not initialised") - return + if u.store != nil { + address = strings.ToLower(address) + return u.store.GetAddressID(address) } - return u.store.GetAddressID(address) + addresses := u.client().Addresses() + pmapiAddress := addresses.ByEmail(address) + if pmapiAddress != nil { + return pmapiAddress.ID, nil + } + return "", errors.New("address not found") } // GetBridgePassword returns bridge password. This is not a password of the PM diff --git a/internal/users/users.go b/internal/users/users.go index f92c4cfb..95ad0402 100644 --- a/internal/users/users.go +++ b/internal/users/users.go @@ -52,8 +52,14 @@ type Users struct { // People are used to that and so we preserve that ordering here. users []*User - // idleUpdates is a channel which the imap backend listens to and which it uses - // to send idle updates to the mail client (eg thunderbird). + // useOnlyActiveAddresses determines whether credentials keeps only active + // addresses or all of them. Each usage has to be consisteng, e.g., once + // user is added, it saves address list to credentials and next time loads + // as is, without requesting server again. + useOnlyActiveAddresses bool + + // idleUpdates is a channel which the imap backend listens to and which it + // uses to send idle updates to the mail client (eg thunderbird). // The user stores should send idle updates on this channel. idleUpdates chan imapBackend.Update @@ -70,19 +76,21 @@ func New( clientManager ClientManager, credStorer CredentialsStorer, storeFactory StoreMaker, + useOnlyActiveAddresses bool, ) *Users { log.Trace("Creating new users") u := &Users{ - config: config, - panicHandler: panicHandler, - events: eventListener, - clientManager: clientManager, - credStorer: credStorer, - storeFactory: storeFactory, - idleUpdates: make(chan imapBackend.Update), - lock: sync.RWMutex{}, - stopAll: make(chan struct{}), + config: config, + panicHandler: panicHandler, + events: eventListener, + clientManager: clientManager, + credStorer: credStorer, + storeFactory: storeFactory, + useOnlyActiveAddresses: useOnlyActiveAddresses, + idleUpdates: make(chan imapBackend.Update), + lock: sync.RWMutex{}, + stopAll: make(chan struct{}), } go func() { @@ -291,9 +299,15 @@ func (u *Users) addNewUser(apiUser *pmapi.User, auth *pmapi.Auth, hashedPassphra return errors.Wrap(err, "failed to update API user") } - activeEmails := client.Addresses().ActiveEmails() + emails := []string{} + for _, address := range client.Addresses() { + if u.useOnlyActiveAddresses && address.Receive != pmapi.CanReceive { + continue + } + emails = append(emails, address.Email) + } - if _, err = u.credStorer.Add(apiUser.ID, apiUser.Name, auth.GenToken(), hashedPassphrase, activeEmails); err != nil { + if _, err = u.credStorer.Add(apiUser.ID, apiUser.Name, auth.GenToken(), hashedPassphrase, emails); err != nil { return errors.Wrap(err, "failed to add user to credentials store") } diff --git a/internal/users/users_test.go b/internal/users/users_test.go index 997a700f..f79fe162 100644 --- a/internal/users/users_test.go +++ b/internal/users/users_test.go @@ -238,7 +238,7 @@ func testNewUsers(t *testing.T, m mocks) *Users { //nolint[unparam] m.eventListener.EXPECT().Add(events.UpgradeApplicationEvent, gomock.Any()) m.clientManager.EXPECT().GetAuthUpdateChannel().Return(make(chan pmapi.ClientAuth)) - users := New(m.config, m.PanicHandler, m.eventListener, m.clientManager, m.credentialsStore, m.storeMaker) + users := New(m.config, m.PanicHandler, m.eventListener, m.clientManager, m.credentialsStore, m.storeMaker, true) waitForEvents() diff --git a/pkg/config/config.go b/pkg/config/config.go index d0432564..f60f0969 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -228,6 +228,11 @@ func (c *Config) GetPreferencesPath() string { return filepath.Join(c.appDirsVersion.UserCache(), "prefs.json") } +// GetTransferDir returns folder for import/export rule and report files. +func (c *Config) GetTransferDir() string { + return filepath.Join(c.appDirsVersion.UserCache()) +} + // GetDefaultAPIPort returns default Bridge local API port. func (c *Config) GetDefaultAPIPort() int { return 1042 diff --git a/pkg/message/body.go b/pkg/message/body.go index 283fc7f3..a704d1e0 100644 --- a/pkg/message/body.go +++ b/pkg/message/body.go @@ -55,7 +55,7 @@ func WriteAttachmentBody(w io.Writer, kr *crypto.KeyRing, m *pmapi.Message, att dr = r err = nil att.Name += ".gpg" - att.MIMEType = "application/pgp-encrypted" + att.MIMEType = "application/pgp-encrypted" //nolint } else if err != nil && err != openpgperrors.ErrSignatureExpired { err = fmt.Errorf("cannot decrypt attachment: %v", err) return diff --git a/pkg/message/build.go b/pkg/message/build.go new file mode 100644 index 00000000..f25cd09b --- /dev/null +++ b/pkg/message/build.go @@ -0,0 +1,348 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package message + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + "io/ioutil" + "mime/multipart" + "mime/quotedprintable" + "net/textproto" + + "github.com/ProtonMail/gopenpgp/crypto" + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/emersion/go-textwrapper" + openpgperrors "golang.org/x/crypto/openpgp/errors" +) + +// Builder for converting PM message to RFC822. Builder will directly write +// changes to message when fetching or building message. +type Builder struct { + cl pmapi.Client + msg *pmapi.Message + + EncryptedToHTML bool + succDcrpt bool +} + +// NewBuilder initiated with client and message meta info. +func NewBuilder(client pmapi.Client, message *pmapi.Message) *Builder { + return &Builder{cl: client, msg: message, EncryptedToHTML: true, succDcrpt: false} +} + +// fetchMessage will update original PM message if successful +func (bld *Builder) fetchMessage() (err error) { + if bld.msg.Body != "" { + return nil + } + + complete, err := bld.cl.GetMessage(bld.msg.ID) + if err != nil { + return + } + + *bld.msg = *complete + + return +} + +func (bld *Builder) writeMessageBody(w io.Writer) error { + if err := bld.fetchMessage(); err != nil { + return err + } + + err := bld.WriteBody(w) + if err != nil { + _, _ = io.WriteString(w, "\r\n") + if bld.EncryptedToHTML { + _ = CustomMessage(bld.msg, err, true) + } + _, err = io.WriteString(w, bld.msg.Body) + _, _ = io.WriteString(w, "\r\n") + } + + return err +} + +func (bld *Builder) writeAttachmentBody(w io.Writer, att *pmapi.Attachment) error { + // Retrieve encrypted attachment + r, err := bld.cl.GetAttachment(att.ID) + if err != nil { + return err + } + defer r.Close() //nolint[errcheck] + + if err := bld.WriteAttachmentBody(w, att, r); err != nil { + // Returning an error here makes e-mail clients like Thunderbird behave + // badly, trying to retrieve the message again and again + log.Warnln("Cannot write attachment body:", err) + } + return nil +} + +func (bld *Builder) writeRelatedPart(p io.Writer, inlines []*pmapi.Attachment) error { + related := multipart.NewWriter(p) + + _ = related.SetBoundary(GetRelatedBoundary(bld.msg)) + + buf := &bytes.Buffer{} + if err := bld.writeMessageBody(buf); err != nil { + return err + } + + // Write the body part + h := GetBodyHeader(bld.msg) + + var err error + if p, err = related.CreatePart(h); err != nil { + return err + } + + _, _ = buf.WriteTo(p) + + for _, inline := range inlines { + buf = &bytes.Buffer{} + if err = bld.writeAttachmentBody(buf, inline); err != nil { + return err + } + + h := GetAttachmentHeader(inline) + if p, err = related.CreatePart(h); err != nil { + return err + } + _, _ = buf.WriteTo(p) + } + + _ = related.Close() + return nil +} + +// BuildMessage converts PM message to body structure (not RFC3501) and bytes +// of RC822 message. If successful the original PM message will contain decrypted body. +func (bld *Builder) BuildMessage() (structure *BodyStructure, message []byte, err error) { //nolint[funlen] + if err = bld.fetchMessage(); err != nil { + return nil, nil, err + } + + bodyBuf := &bytes.Buffer{} + + mainHeader := GetHeader(bld.msg) + mainHeader.Set("Content-Type", "multipart/mixed; boundary="+GetBoundary(bld.msg)) + if err = WriteHeader(bodyBuf, mainHeader); err != nil { + return nil, nil, err + } + _, _ = io.WriteString(bodyBuf, "\r\n") + + // NOTE: Do we really need extra encapsulation? i.e. Bridge-IMAP message is always multipart/mixed + + if bld.msg.MIMEType == pmapi.ContentTypeMultipartMixed { + _, _ = io.WriteString(bodyBuf, "\r\n--"+GetBoundary(bld.msg)+"\r\n") + if err = bld.writeMessageBody(bodyBuf); err != nil { + return nil, nil, err + } + _, _ = io.WriteString(bodyBuf, "\r\n--"+GetBoundary(bld.msg)+"--\r\n") + } else { + mw := multipart.NewWriter(bodyBuf) + _ = mw.SetBoundary(GetBoundary(bld.msg)) + + var partWriter io.Writer + atts, inlines := SeparateInlineAttachments(bld.msg) + + if len(inlines) > 0 { + relatedHeader := GetRelatedHeader(bld.msg) + if partWriter, err = mw.CreatePart(relatedHeader); err != nil { + return nil, nil, err + } + _ = bld.writeRelatedPart(partWriter, inlines) + } else { + buf := &bytes.Buffer{} + if err = bld.writeMessageBody(buf); err != nil { + return nil, nil, err + } + + // Write the body part + bodyHeader := GetBodyHeader(bld.msg) + if partWriter, err = mw.CreatePart(bodyHeader); err != nil { + return nil, nil, err + } + + _, _ = buf.WriteTo(partWriter) + } + + // Write the attachments parts + for _, att := range atts { + buf := &bytes.Buffer{} + if err = bld.writeAttachmentBody(buf, att); err != nil { + return nil, nil, err + } + + attachmentHeader := GetAttachmentHeader(att) + if partWriter, err = mw.CreatePart(attachmentHeader); err != nil { + return nil, nil, err + } + + _, _ = buf.WriteTo(partWriter) + } + + _ = mw.Close() + } + + // wee need to copy buffer before building body structure + message = bodyBuf.Bytes() + structure, err = NewBodyStructure(bodyBuf) + return structure, message, err +} + +// SuccessfullyDecrypted is true when message was fetched and decrypted successfully +func (bld *Builder) SuccessfullyDecrypted() bool { return bld.succDcrpt } + +// WriteBody decrypts PM message and writes main body section. The external PGP +// message is written as is (including attachments) +func (bld *Builder) WriteBody(w io.Writer) error { + kr, err := bld.cl.KeyRingForAddressID(bld.msg.AddressID) + if err != nil { + return err + } + // decrypt body + if err := bld.msg.Decrypt(kr); err != nil && err != openpgperrors.ErrSignatureExpired { + return err + } + bld.succDcrpt = true + if bld.msg.MIMEType != pmapi.ContentTypeMultipartMixed { + // transfer encoding + qp := quotedprintable.NewWriter(w) + if _, err := io.WriteString(qp, bld.msg.Body); err != nil { + return err + } + return qp.Close() + } + _, err = io.WriteString(w, bld.msg.Body) + return err +} + +// WriteAttachmentBody decrypts and writes the attachments +func (bld *Builder) WriteAttachmentBody(w io.Writer, att *pmapi.Attachment, attReader io.Reader) (err error) { + kr, err := bld.cl.KeyRingForAddressID(bld.msg.AddressID) + if err != nil { + return err + } + // Decrypt it + var dr io.Reader + dr, err = att.Decrypt(attReader, kr) + if err == openpgperrors.ErrKeyIncorrect { + // Do not fail if attachment is encrypted with a different key + dr = attReader + err = nil + att.Name += ".gpg" + att.MIMEType = "application/pgp-encrypted" + } else if err != nil && err != openpgperrors.ErrSignatureExpired { + err = fmt.Errorf("cannot decrypt attachment: %v", err) + return err + } + + // transfer encoding + ww := textwrapper.NewRFC822(w) + bw := base64.NewEncoder(base64.StdEncoding, ww) + + var n int64 + if n, err = io.Copy(bw, dr); err != nil { + err = fmt.Errorf("cannot write attachment: %v (wrote %v bytes)", err, n) + } + + _ = bw.Close() + return err +} + +func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) ([]byte, error) { //nolint[funlen] + b := &bytes.Buffer{} + + // Overwrite content for main header for import. + // Even if message has just simple body we should upload as multipart/mixed. + // Each part has encrypted body and header reflects the original header. + mainHeader := GetHeader(m) + mainHeader.Set("Content-Type", "multipart/mixed; boundary="+GetBoundary(m)) + mainHeader.Del("Content-Disposition") + mainHeader.Del("Content-Transfer-Encoding") + if err := WriteHeader(b, mainHeader); err != nil { + return nil, err + } + mw := multipart.NewWriter(b) + if err := mw.SetBoundary(GetBoundary(m)); err != nil { + return nil, err + } + + // Write the body part. + bodyHeader := make(textproto.MIMEHeader) + bodyHeader.Set("Content-Type", m.MIMEType+"; charset=utf-8") + bodyHeader.Set("Content-Disposition", "inline") + bodyHeader.Set("Content-Transfer-Encoding", "7bit") + + p, err := mw.CreatePart(bodyHeader) + if err != nil { + return nil, err + } + // First, encrypt the message body. + if err := m.Encrypt(kr, kr); err != nil { + return nil, err + } + if _, err := io.WriteString(p, m.Body); err != nil { + return nil, err + } + + // Write the attachments parts. + for i := 0; i < len(m.Attachments); i++ { + att := m.Attachments[i] + r := readers[i] + h := GetAttachmentHeader(att) + p, err := mw.CreatePart(h) + if err != nil { + return nil, err + } + // Create line wrapper writer. + ww := textwrapper.NewRFC822(p) + + // Create base64 writer. + bw := base64.NewEncoder(base64.StdEncoding, ww) + + data, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + // Create encrypted writer. + pgpMessage, err := kr.Encrypt(crypto.NewPlainMessage(data), nil) + if err != nil { + return nil, err + } + if _, err := bw.Write(pgpMessage.GetBinary()); err != nil { + return nil, err + } + if err := bw.Close(); err != nil { + return nil, err + } + } + + if err := mw.Close(); err != nil { + return nil, err + } + + return b.Bytes(), nil +} diff --git a/pkg/message/message.go b/pkg/message/message.go index 2115112c..6093c180 100644 --- a/pkg/message/message.go +++ b/pkg/message/message.go @@ -23,8 +23,11 @@ import ( "strings" "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/sirupsen/logrus" ) +var log = logrus.WithField("pkg", "pkg/message") //nolint[gochecknoglobals] + func GetBoundary(m *pmapi.Message) string { // The boundary needs to be deterministic because messages are not supposed to // change. diff --git a/pkg/message/parser.go b/pkg/message/parser.go index ee7deea1..80da7ab8 100644 --- a/pkg/message/parser.go +++ b/pkg/message/parser.go @@ -37,7 +37,6 @@ import ( pmmime "github.com/ProtonMail/proton-bridge/pkg/mime" "github.com/ProtonMail/proton-bridge/pkg/pmapi" "github.com/jaytaylor/html2text" - log "github.com/sirupsen/logrus" ) func parseAttachment(filename string, mediaType string, h textproto.MIMEHeader) (att *pmapi.Attachment) { diff --git a/pkg/message/utils.go b/pkg/message/utils.go new file mode 100644 index 00000000..c66fb514 --- /dev/null +++ b/pkg/message/utils.go @@ -0,0 +1,85 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package message + +import ( + "bytes" + "html/template" + "io" + "net/http" + "net/mail" + "net/textproto" + + "github.com/ProtonMail/proton-bridge/pkg/pmapi" +) + +func WriteHeader(w io.Writer, h textproto.MIMEHeader) (err error) { + if err = http.Header(h).Write(w); err != nil { + return + } + _, err = io.WriteString(w, "\r\n") + return +} + +const customMessageTemplate = ` + + + +
+ Decryption error
+ Decryption of this message's encrypted content failed. + 
{{.Error}}
+
+ + {{if .AttachBody}} +
+ 
{{.Body}}
+
+ {{- end}} + + +` + +type customMessageData struct { + Error string + AttachBody bool + Body string +} + +func CustomMessage(m *pmapi.Message, decodeError error, attachBody bool) error { + t := template.Must(template.New("customMessage").Parse(customMessageTemplate)) + + b := new(bytes.Buffer) + + if err := t.Execute(b, customMessageData{ + Error: decodeError.Error(), + AttachBody: attachBody, + Body: m.Body, + }); err != nil { + return err + } + + m.MIMEType = pmapi.ContentTypeHTML + m.Body = b.String() + + // NOTE: we need to set header in custom message header, so we check that is non-nil. + if m.Header == nil { + m.Header = make(mail.Header) + } + return nil +} diff --git a/pkg/pmapi/clientmanager.go b/pkg/pmapi/clientmanager.go index c4e44d80..b351347f 100644 --- a/pkg/pmapi/clientmanager.go +++ b/pkg/pmapi/clientmanager.go @@ -347,6 +347,14 @@ func (cm *ClientManager) CheckConnection() error { return nil } +// CheckConnection returns an error if there is no internet connection. +func CheckConnection() error { + client := &http.Client{Timeout: time.Second * 10} + retStatus := make(chan error) + checkConnection(client, "http://protonstatus.com/vpn_status", retStatus) + return <-retStatus +} + func checkConnection(client *http.Client, url string, errorChannel chan error) { resp, err := client.Get(url) if err != nil { diff --git a/pkg/pmapi/import.go b/pkg/pmapi/import.go index 2a7f9d96..804b464d 100644 --- a/pkg/pmapi/import.go +++ b/pkg/pmapi/import.go @@ -95,6 +95,11 @@ type ImportMsgReq struct { LabelIDs []string } +func (req ImportMsgReq) String() string { + data, _ := json.Marshal(req) + return string(data) +} + // ImportRes is a response to an import request. type ImportRes struct { Res diff --git a/pkg/updates/updates.go b/pkg/updates/updates.go index de58ce69..2b0a9607 100644 --- a/pkg/updates/updates.go +++ b/pkg/updates/updates.go @@ -120,7 +120,7 @@ func (u *Updates) CreateJSONAndSign(deployDir, goos string) error { return nil } -func (u *Updates) CheckIsBridgeUpToDate() (isUpToDate bool, latestVersion VersionInfo, err error) { +func (u *Updates) CheckIsUpToDate() (isUpToDate bool, latestVersion VersionInfo, err error) { localVersion := u.GetLocalVersion() latestVersion, err = u.getLatestVersion() if err != nil { diff --git a/pkg/updates/updates_test.go b/pkg/updates/updates_test.go index 4482c0e2..220864d1 100644 --- a/pkg/updates/updates_test.go +++ b/pkg/updates/updates_test.go @@ -71,14 +71,14 @@ func startServer() { func TestCheckBridgeIsUpToDate(t *testing.T) { updates := newTestUpdates("1.1.6") - isUpToDate, _, err := updates.CheckIsBridgeUpToDate() + isUpToDate, _, err := updates.CheckIsUpToDate() require.NoError(t, err) require.True(t, isUpToDate, "Bridge should be up to date") } func TestCheckBridgeIsNotUpToDate(t *testing.T) { updates := newTestUpdates("1.1.5") - isUpToDate, _, err := updates.CheckIsBridgeUpToDate() + isUpToDate, _, err := updates.CheckIsUpToDate() require.NoError(t, err) require.True(t, !isUpToDate, "Bridge should not be up to date") } diff --git a/release-notes/bugs.txt b/release-notes/bugs-bridge.txt similarity index 100% rename from release-notes/bugs.txt rename to release-notes/bugs-bridge.txt diff --git a/release-notes/bugs-importexport.txt b/release-notes/bugs-importexport.txt new file mode 100644 index 00000000..e69de29b diff --git a/release-notes/notes.txt b/release-notes/notes-bridge.txt similarity index 100% rename from release-notes/notes.txt rename to release-notes/notes-bridge.txt diff --git a/release-notes/notes-importexport.txt b/release-notes/notes-importexport.txt new file mode 100644 index 00000000..e69de29b diff --git a/utils/credits.sh b/utils/credits.sh index 7943c1ad..6827c7b0 100755 --- a/utils/credits.sh +++ b/utils/credits.sh @@ -1,6 +1,5 @@ #!/bin/bash - # Copyright (c) 2020 Proton Technologies AG # # This file is part of ProtonMail Bridge. @@ -20,6 +19,8 @@ ## Generate credits from go.mod +PACKAGE=$1 + # Vendor packages LOCKFILE=../go.mod egrep $'^\t[^=>]*$' $LOCKFILE | sed -r 's/\t([^ ]*) v.*/\1/g' > tmp1 @@ -30,6 +31,6 @@ echo -e "\nFont Awesome 4.7.0\n\nQt 5.13 by Qt group\n" >> tmp # join lines sed -i -e ':a' -e 'N' -e '$!ba' -e 's|\n|;|g' tmp -cat ../utils/license_header.txt > ../internal/bridge/credits.go -echo -e '// Code generated by '`echo $0`' at '`date`'. DO NOT EDIT.\n\npackage bridge\n\nconst Credits = "'$(cat tmp)'"' >> ../internal/bridge/credits.go +cat ../utils/license_header.txt > ../internal/$PACKAGE/credits.go +echo -e '// Code generated by '`echo $0`' at '`date`'. DO NOT EDIT.\n\npackage '$PACKAGE'\n\nconst Credits = "'$(cat tmp)'"' >> ../internal/$PACKAGE/credits.go rm tmp1 tmp diff --git a/utils/release-notes.sh b/utils/release-notes.sh index 019afdde..9ab5ff41 100755 --- a/utils/release-notes.sh +++ b/utils/release-notes.sh @@ -17,7 +17,8 @@ # You should have received a copy of the GNU General Public License # along with ProtonMail Bridge. If not, see . +PACKAGE=$1 # Generate release notes information -cat ../utils/license_header.txt > ../internal/bridge/release_notes.go -echo -e '// Code generated by '`echo $0`' at '`date`'. DO NOT EDIT.\n\npackage bridge\n\nconst ReleaseNotes = `'"$(cat ../release-notes/notes.txt)"'\n`\n\nconst ReleaseFixedBugs = `'"$(cat ../release-notes/bugs.txt)"'\n`' >> ../internal/bridge/release_notes.go +cat ../utils/license_header.txt > ../internal/$PACKAGE/release_notes.go +echo -e "// Code generated by `echo $0` at '`date`'. DO NOT EDIT.\n\npackage ${PACKAGE}\n\nconst ReleaseNotes = \``cat ../release-notes/notes-${PACKAGE}.txt`\n\`\n\nconst ReleaseFixedBugs = \``cat ../release-notes/bugs-${PACKAGE}.txt`\n\`" >> ../internal/$PACKAGE/release_notes.go