babcom_cli

#!/usr/bin/env python3
# encoding: utf-8

# babcom.py --- Implementation of Freenet Communication Primitives

# Copyright (C) 2016 Arne Babenhauserheide <arne_bab@web.de>
# Copyright (C) 2013 Steve Dougherty (Freemail and WoT concepts)

# Author: Arne Babenhauserheide <arne_bab@web.de>

# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.


"""Implementation of Freenet Commmunication Primitives"""


import sys
import os
import glob
import time
import datetime
import shutil
import stat
import zipfile
import urllib.request
import subprocess
import argparse # commandline arguments
import cmd # interactive shell
import getpass
import fcp3 as fcp
import freenet3 as freenet
import freenet3.appdirs as appdirs
import random
import threading # TODO: replace by futures for Python3
import logging
import functools
import hashlib
import smtplib
from email.mime.text import MIMEText
import imaplib
import base64
import re
try:
    import passlib.hash as passlib_hash
except ImportError:
    import freenet_passlib_170.hash as passlib_hash

try:
    import newbase60
    numtostring = newbase60.numtosxg
except Exception:
    numtostring = str

if "--debug" in sys.argv:
    logging.basicConfig(level=logging.DEBUG,
                        format=' [%(levelname)-7s] (%(asctime)s) %(filename)s::%(lineno)d %(message)s',
                        datefmt='%Y-%m-%d %H:%M:%S')
else:
    logging.basicConfig(level=logging.WARNING,
                        format=' [%(levelname)-7s] %(message)s',
                        datefmt='%Y-%m-%d %H:%M:%S')
    
    
PY3 = sys.version_info > (3,)

slowtests = False

in_doctest = False

# first, parse commandline arguments
def parse_args():
    """Parse commandline arguments."""
    parser = argparse.ArgumentParser(description="Implementation of Freenet Communication Primitives")
    parser.add_argument('-u', '--user', default=None, help="Identity to use (default: create new)")
    parser.add_argument('--recover', action='store_true', help="Recover an identity from its recovery secret")
    parser.add_argument('--host', default=None, help="Freenet host address (default: 127.0.0.1)")
    parser.add_argument('--port', default=None, help="Freenet FCP port (default: random, or 9481 if using --no-spawn)")
    parser.add_argument('--verbosity', default=None, help="Set verbosity. For tests any verbosity activates verbose tests (default: 3, to FCP calls: 5)")
    parser.add_argument('--debug', action="store_true", help="Show debug messages and use more debug-friendly output.")
    parser.add_argument('--transient', default=False, action="store_true", help="Do not store any data in babcom (to preserve data in the node, use --no-spawn)")
    parser.add_argument('--no-spawn', default=False, action="store_true", help="Do not spawn a Freenet node: Use an existing node at the given --port instead.")
    parser.add_argument('--spawn', default=True, action="store_true", help="Spawn a freenet node. If the fcp-port is given, and no node is active at that port, spawn with the given FCP port. Otherwise spawn with a random port (option active by default, kept for backwards compatibility. Use --no-spawn to disable).")
    parser.add_argument('--test', default=False, action="store_true", help="Run the tests")
    parser.add_argument('--slowtests', default=False, action="store_true", help="Run slow tests, many of them with actual network operation in Freenet")
    args = parser.parse_args()
    return args


# then prepare giving progress
def withprogress(func):
    """Provide progress, if we are the main thread (blocking others)"""
    @functools.wraps(func)
    def fun(*args, **kwds):
        # avoid giving progress when we’re not the main thread
        if not isinstance(threading.current_thread(), threading._MainThread):
            return func(*args, **kwds)
        # also avoid progress when running in doctests, from https://stackoverflow.com/a/22734497
        if in_doctest:
            return func(*args, **kwds)
        
        def waiting(letter):
            def w():
                sys.stderr.write(letter)
                sys.stderr.flush()
            return w
        
        def funcinfo(fun):
            _n = func.__name__
            return "".join(["[",
                            _n[:2],
                            hashlib.sha256(_n.encode("utf-8")).hexdigest()[:1],
                            _n[-2:],
                            "]"])
        tasks = []
        # start with the function name
        tasks.append(threading.Timer(0.9, waiting(funcinfo(func))))
        # one per second for 20 seconds
        for i in range(1, 21):
            tasks.append(threading.Timer(i, waiting(".")))
        # one per 3 seconds for 1 minute
        for i in range(1, 21):
            tasks.append(threading.Timer(20 + i*3, waiting(":")))
        # one per 10 seconds for 3.5 minutes
        for i in range(1, 22):
            tasks.append(threading.Timer(80 + i*10, waiting("#")))
        # one per 5 minutes for the rest of the hour
        for i in range(1, 12):
            tasks.append(threading.Timer(300 + i*300, waiting("!")))
        # one per hour for 6 hours
        for i in range(1, 7):
            tasks.append(threading.Timer(3600 + i*3600, waiting("?")))
        [i.start() for i in tasks]
        try:
            res = func(*args, **kwds)
        except Exception:
            raise
        finally:
            [i.cancel() for i in tasks]
            sys.stderr.write("\n")
            sys.stderr.flush()
        return res

    return fun


# and output that does not disturb doctests


# then add interactive usage, since this will be a communication tool
class Babcom(cmd.Cmd):
    # attributes to be changed from outside
    recover = False
    username = None
    identity = None
    requestkey = None
    recoverysecret0 = None
    recoverysecret1 = None
    recoverysecret2 = None
    transient = False # in transient operation, nothing is saved to disk
    spawn = False   # was the node spawned for this run? Transient
                    # spawns are destroyed on quit.
    fcp_port = None # needed for spawns to track down their folder
    prompt = "--> "
    # internal attributes
    finished_startup = False
    _messageprompt = "{newidentities}{messages}> "
    _emptyprompt = "--> "
    # TODO: change to "!5> " for 5 messages which can then be checked
    #       with the command read.
    #: seed identity keys for initial visibility. This is currently BabcomTest. They need to be maintained: a daemon needs to actually check their CAPTCHA queue and update the trust, and a human needs to check whether what they post is spam or not.
    seedkeys = [
        # ("USK@fVzf7fg0Va7vNTZZQNKMCDGo6-FSxzF3PhthcXKRRvA,"
        #  "~JBPb2wjAfP68F1PVriphItLMzioBP9V9BnN59mYb0o,"
        #  "AQACAAE/WebOfTrust/12"),
        ("USK@IUqsUoSRuuJgEk7CkFWyXRK4fSffYZGlzPWxcHlLYWM,nbI93BRVXKvXsAwB5112i8aC09fwo6Md1nju6Jg5nHs,AQACAAE/WebOfTrust/0")
    ]
    seedtrust = 100
    # iterators which either return a CAPTCHA or None.
    captchaiters = [] # one iterator per set of captcha-sources to check
    captchas = [] # retrieved captchas I could solve
    captchawatchers = [] # one iterator per set of captchasolutionkeys
    captchasolutions = [] # captcha solutions to watch for new identities
    newlydiscovered = [] # newly discovered IDs
    messages = [] # new messages the user can read
    timers = []
    nod_own = None # news of the day, own
    nod_shared = None # news of the day
    nods = None # news of the day to read

    def preloop(self):
        creatednewid = False
        if self.recover:
            if self.username:
                print("Specified both username and recovery. Ignoring username in favor of recovery.")
            # TOOD: Upload recovery information. Create and show the secret.
            # load the secret user information from Freenet
            print("Recovering identity, please type your recovery secret.")
            recoverysecret = getpass.getpass("Recovery Secret: ").strip()
            self.recoverysecret0, self.recoverysecret1, self.recoverysecret2 = split_recovery_secret_string(recoverysecret)
            logging.info("Waiting until Freenet has at least five connections...")
            
            def realpeers(n):
                return [i for i in n.listpeers() if "seed" in i and i["seed"] == "false"]
            with fcp.FCPNode() as n:
                while len(realpeers(n)) < 5:
                    time.sleep(1)
            print("... retrieving recovery information...")
            try:
                username, self.identity = recover_request_key_and_name(
                    self.recoverysecret0, self.recoverysecret1).split(b"@")
            except fcp.FCPGetFailed as e:
                print("could not retrieve the recovery information from Freenet. Please check your recovery secret. Recovery should work for at least 4 weeks after the last insert.")
                raise
            with fcp.FCPNode() as n:
                self.insertkey = recover_insert_key(self.recoverysecret2, self.identity, username)
                print("Creating a local version of your ID with your recovered username and insert key...")
                try:
                    self.username = createidentity(name=username, insertkey=self.insertkey)
                except fcp.ProtocolError as e:
                    # InvalidParameterException means that an ID with this key already exists. We can simply grab the matching username in the next step.
                    if str(e).startswith("plugins.WebOfTrust.exceptions.InvalidParameterException"):
                        print("An ID with this insert key is already in the local Web of Trust. Will chose the existing one in the next step.")
                    else:
                        raise
        elif self.username is None:
            print("No user given, creating random name...")
            self.username = randomname()
            print("... generating random identity with name", self.username, ". Please wait...")
        else:
            print("Retrieving identity information from Freenet using name", self.username + ". Please wait ...")

        if not in_doctest:
            print("... checking for existing identities which match", self.username, "...")
        matches = getownidentities(self.username)
        if not matches:
            if not in_doctest:
                print("... no matches found, creating new identity...")
            createidentity(self.username)
            creatednewid = True
            print("... created identity. Please type down the following recovery secret to be able to recover the identity from another computer:")
            self.recoverysecret0 = str(time.gmtime()[0])
            self.recoverysecret1 = create_recovery_secret_part(2) # entropy ~ 44
            self.recoverysecret2 = create_recovery_secret_part(3) # entropy ~ 72
            print(join_recovery_secret_string(self.recoverysecret0, self.recoverysecret1, self.recoverysecret2))
            matches = getownidentities(self.username)
        
        print("... retrieved", len(matches), "identities matching", self.username)
        if matches[1:]:
            choice = None
            print("more than one identity with name", self.username, "please select one.")
            while choice is None:
                for i in range(len(matches)):
                    print(i+1, matches[i][0]+"@"+matches[i][1]["Identity"])
                res = input("Insert the number of the identity to use (1 to " + str(len(matches)) + "): ")
                try:
                    choice = int(res)
                except ValueError:
                    print("not a number")
                if choice < 1 or len(matches) < choice:
                    choice = None
                    print("the number is not in the range", str(i+1), "to", str(len(matches)))
            self.username = matches[choice - 1][0]
            self.identity = matches[choice - 1][1]["Identity"]
            self.requestkey = matches[choice - 1][1]["RequestURI"]
        else:
            self.username = matches[0][0]
            self.identity = matches[0][1]["Identity"]
            self.requestkey = matches[0][1]["RequestURI"]

        # load persistent state from disk (i.e. unused captcha solutions)
        self.load()
        
        print("Logged in as", self.username + "@" + self.identity)
        print("    with key", self.requestkey)
        if creatednewid:
            print("Uploading recovery information using the recovery secret",
                  join_recovery_secret_string(self.recoverysecret0, self.recoverysecret1, self.recoverysecret2))
            upload_recovery(self.recoverysecret0, self.recoverysecret1, self.recoverysecret2,
                            self.identity, self.username,
                            captchasolutions="\n".join(self.captchasolutions))
        # start watching captcha solutions
        self.watchcaptchasolutionloop()
        
        def introduce():
            # TODO: write solutions to a file on disk and re-read a
            # limited number of them on login.
            solutions = providecaptchas(self.identity)
            self.captchasolutions.extend(solutions)
            self.watchcaptchasolutions(solutions)
            self.messages.append("New CAPTCHAs uploaded successfully.")
            print("\a", end="") # alert / bell FIXME: does not work yet.
        t = threading.Timer(0, introduce)
        t.daemon = True
        t.start()
        self.timers.append(t)
        print("Providing new CAPTCHAs, so others can make themselves visible.""")
        print()
        self.finished_startup = True

    def postloop(self):
        """Cleanup and save state."""
        # TODO: This does not fire on quit or EOF. The functionality
        #       is for now implemented in do_quit and do_EOF.

    def postcmd(self, stop, line):
        # update message information after every command
        self.updateprompt()

    def cmdloop(self, *args, **kwds):
        try:
            super().cmdloop(*args, **kwds)
        except KeyboardInterrupt as e:
            if not self.finished_startup:
                raise # if we do not have a command loop yet where the
                      # user can exit, actually do go down on CTRL-C.
            logging.warning("Caught Keyboard Interrupt (CTRL-C). Restarting commandloop. Use CTRL-D to exit.")
            self.cmdloop(*args, **kwds)

    def teardown(self):
        """Delete the node folder. Only for spawns!"""
        if not self.spawn:
            logging.warning("Tried to teardown a node which is no spawn")
            return
        if self.transient:
            print("Stopping and deleting the spawned Freenet node.")
        return freenet.spawn.teardown_node(self.fcp_port,
                                           delete_node_folder=self.transient)
        
    def save(self):
        """Save state like the CAPTCHA solutions."""
        if self.transient:
            return # not saving anything in transient mode
        
        dirs = appdirs.AppDirs("babcom", "freenet")
        # dirs.user_data_dir 
        # dirs.user_config_dir 
        # dirs.user_cache_dir
        # dirs.user_log_dir
        identity_info_dir = os.path.join(dirs.user_data_dir, self.identity)
        if not os.path.isdir(identity_info_dir):
            os.makedirs(identity_info_dir)
        with open(os.path.join(
                identity_info_dir, "unusedcaptchasolutions.txt"), "w") as f:
            f.write("\n".join(self.captchasolutions))
        if None in (self.recoverysecret0, self.recoverysecret1, self.recoverysecret2):
            self.createrecovery()
            upload_recovery(self.recoverysecret0, self.recoverysecret1, self.recoverysecret2,
                            self.identity, self.username,
                            captchasolutions="\n".join(self.captchasolutions))
        with open(os.path.join(
                identity_info_dir, "recoverysecret.txt"), "w") as f:
            f.write(join_recovery_secret_string(
                self.recoverysecret0, self.recoverysecret1, self.recoverysecret2))
        # add the user to the known users to be able to recover it.
        knownusersfile = os.path.join(dirs.user_data_dir, "knownusers")
        adduser = False
        if not os.path.isfile(knownusersfile):
            adduser = True
        else:
            with open(knownusersfile) as f:
                if self.identity not in f:
                    adduser = True
        if adduser:
            with open(knownusersfile, "a") as f:
                f.write("{} {} {}\n".format(self.identity, self.fcp_port, self.username))
                logging.info("saved user {} as new known user with id {} and port {}".format(
                    self.username, self.identity, self.fcp_port))
        logging.debug("saved recoverysecret and {}".format(self.captchasolutions))
        
    def load(self):
        """Save state like the CAPTCHA solutions."""
        dirs = appdirs.AppDirs("babcom", "freenet")
        identity_info_dir = os.path.join(dirs.user_data_dir, self.identity)
        if not os.path.isdir(identity_info_dir):
            if None in (self.recoverysecret0, self.recoverysecret1, self.recoverysecret2):
                print("... no recovery secret found, creating and uploading a new one...")
                recoverysecret = self.createrecovery()
                upload_recovery(self.recoverysecret0, self.recoverysecret1, self.recoverysecret2,
                                self.identity, self.username,
                                captchasolutions="\n".join(self.captchasolutions))
                print("... please write down the following recovery secret:")
                print(recoverysecret)
            return

        with open(os.path.join(
                identity_info_dir, "unusedcaptchasolutions.txt")) as f:
            solutions = [i.strip() for i in f.readlines()]
        c = set(self.captchasolutions)
        self.captchasolutions.extend(
            [i for i in solutions if i not in c])
        logging.debug("loaded {}".format(solutions))
        try:
            with open(os.path.join(
                    identity_info_dir, "recoverysecret.txt")) as f:
                self.recoverysecret0, self.recoverysecret1, self.recoverysecret2 = split_recovery_secret_string(
                    f.read().strip())
        except ValueError: # does not exist
            print("... no recovery secret found, creating and uploading a new one...")
            recoverysecret = self.createrecovery()
            upload_recovery(self.recoverysecret0, self.recoverysecret1, self.recoverysecret2,
                            self.identity, self.username,
                            captchasolutions="\n".join(self.captchasolutions))
            print("... please write down the following recovery secret:")
            print(recoverysecret)
            
    def createrecovery(self):
        """Create and remember a recovery secret.

        Returns the secret string.
        """
        self.recoverysecret0 = str(time.gmtime()[0])
        self.recoverysecret1 = create_recovery_secret_part(2) # entropy ~ 44
        self.recoverysecret2 = create_recovery_secret_part(3) # entropy ~ 72
        upload_recovery(self.recoverysecret0, self.recoverysecret1, self.recoverysecret2,
                        self.identity, self.username,
                        captchasolutions="\n".join(self.captchasolutions))
        return join_recovery_secret_string(self.recoverysecret0, self.recoverysecret1, self.recoverysecret2)
        
    def watchcaptchasolutions(self, solutions, maxwatchers=100):
        """Start watching the solutions of captchas, adding trust 0 as needed.

        The real work is done by watchcaptchasolutionsloop.
        """
        # avoid duplicates
        c = set(self.captchasolutions)
        s = [i for i in solutions if i not in c]
        self.captchasolutions.extend(s)
        # never watch more than maxwatchers solutions: drop old entries
        self.captchasolutions = self.captchasolutions[-maxwatchers:]
        # watch the solutions.
        self.captchawatchers.append(watchcaptchas(s))

    def updateprompt(self):
        nummsg = len(self.messages)
        numnew = len(self.newlydiscovered)
        if nummsg + numnew != 0:
            newids = (numtostring(numnew) if numnew > 0 else "!")
            newmsg = (numtostring(nummsg) if nummsg > 0 else "-")
            self.prompt = self._messageprompt.format(newidentities=newids,
                                                     messages=newmsg)
        else:
            self.prompt = self._emptyprompt

    def watchcaptchasolutionloop(self, intervalseconds=300):
        """Watch for captchasolutions in an infinite, offthread loop, adding solutions to newlydiscovered."""
        def loop():
            # resubmit all unsolved captchas if there are no captchasolutions left.
            if not self.captchawatchers and self.captchasolutions:
                self.watchcaptchasolutions(self.captchasolutions)

            for watcher in self.captchawatchers[:]:
                try:
                    res = next(watcher)
                except StopIteration:
                    self.captchawatchers.remove(watcher)
                    continue
                if res is None:
                    continue
                solution, newrequestkey = res
                # remember that the captcha has been solved: do not try again
                try:
                    self.captchasolutions.remove(solution)
                except ValueError: # already removed
                    pass
                newidentity = identityfrom(newrequestkey)
                print(newidentity)
                trustifmissing = 0
                commentifmissing = "Trust received from solving a CAPTCHA"
                trustadded = ensureavailability(newidentity, newrequestkey, self.identity,
                                                trustifmissing=trustifmissing,
                                                commentifmissing=commentifmissing)
                # now the identity is there, but it might not have needed explicit trust.
                # but captchas should give that.
                if not trustadded:
                    trust = gettrust(self.identity, newidentity)
                    if trust == "Nonexistent" or int(trust) < 0:
                        settrust(self.identity, newidentity, trustifmissing, commentifmissing)
                        trustadded = True
                if trustadded:
                    self.newlydiscovered.append(newrequestkey)
                    self.messages.append("New identity added who solved a CAPTCHA: {}".format(newidentity))
                    print("\a", end="") # alert / bell
                else:
                    self.messages.append("Identity {} who solved a CAPTCHA was already known.".format(newidentity))
            
            t = threading.Timer(intervalseconds, loop)
            t.daemon = True
            t.start()
            self.timers.append(t)
            # cleanup the timers
            for t in self.timers:
                if not t.is_alive():
                    t.join()
                    self.timers.remove(t)
        loop()
        
        
    def do_intro(self, *args):
        "Introduce Babcom"
        print("""
> It began in the Earth year 2016, with the founding of the first of
  the Babcom systems, located deep in decentralized space. It was a
  port of call for journalists, writers, hackers, activists . . . and
  travelers from a hundred worlds. Could be a dangerous place – but we
  accepted the risk, because Babcom 1 was societies next, best hope
  for survival.
— Tribute to Babylon 5, where humanity learned to forge its own path.

Type help or help <command> and a newline to learn how to use babcom.

Use introduce to become visible.

If the prompt changes from --> to !M>, N-> or NM>,
   you have new messages. Read them with read
""")
    # for testing: 
    # introduce USK@FpcnriKy19ztmHhg0QzTJjGwEJJ0kG7xgLiOvKXC7JE,CIpXjQej5StQRC8LUZnu3nvvh1l9UbZMinyFQyLSdMY,AQACAAE/WebOfTrust/0
    # introduce USK@0kq3fHCn12-93PSV4kk56B~beIkh-XfjennLapmmapM,9hQr66rxc9O5ptdmfhMk37h2vZGrsE6NYXcFDMGMiTw,AQACAAE/WebOfTrust/1
    # introduce USK@0kq3fHCn12-93PSV4kk56B~beIkh-XfjennLapmmapM,9hQr66rxc9O5ptdmfhMk37h2vZGrsE6NYXcFDMGMiTw,AQACAAE/WebOfTrust/1
    # introduce USK@FZynnK5Ngi6yTkBAZXGbdRLHVPvQbd2poW6DmZT8vbs,bcPW8yREf-66Wfh09yvx-WUt5mJkhGk5a2NFvbCUDsA,AQACAAE/WebOfTrust/1
    # introduce USK@B324z0kMF27IjNEVqn6oRJPJohAP2NRZDFhQngZ1GOI,DRf8JZviHLIFOYOdu42GLL2tDhVaWb6ihdNO18DkTpc,AQACAAE/WebOfTrust/0

    def do_read(self, *args):
        """Read messages."""
        if len(self.messages) + len(self.newlydiscovered) == 0:
            print("No new messages.")
        i = 1
        while self.messages:
            print("[{}]".format(i), end=' ') 
            print(self.messages.pop())
            print()
            i += 1
        if self.newlydiscovered:
            print("discovered {} new identities:".format(len(self.newlydiscovered)))
        i = 1
        while self.newlydiscovered:
            print(i, "-", self.newlydiscovered.pop())
            i += 1
        self.updateprompt()
    
    def do_introduce(self, *args):
        """Introduce your own ID to others. Usage: introduce [<other id key> ...]"""
        usingseeds = args[0] == ""
        if usingseeds and self.captchas:
            return self.onecmd("solvecaptcha")

        def usecaptchas(captchas):
            cap = captchas.splitlines()
            c = set(cap) # avoid duplicates
            # shuffle all new captchas, but not the old ones
            self.captchas = [i for i in self.captchas
                             if i not in c]
            random.shuffle(cap)
            self.captchas.extend(cap)
            return self.onecmd("solvecaptcha")
        
        if usingseeds and self.captchaiters:
            for captchaiter in self.captchaiters[:]:
                try:
                    captchas = next(captchaiter)
                except StopIteration: # captchaiter is finished, nothing more to gain
                    self.captchhaiters.remove(captchaiter)
                else:
                    if captchas is not None:
                        return usecaptchas(captchas)
            return

        if usingseeds:
            ids = [i.split("@")[1].split(",")[0] for i in self.seedkeys]
            keys = self.seedkeys
            trustifmissing = self.seedtrust
            commentifmissing = "Automatically assigned trust to a seed identity."
        else:
            try:
                ids = [i.split("@")[1].split(",")[0] for i in args[0].split()]
            except IndexError:
                print("Invalid id key. Interpreting as ID")
                try:
                    ids = args[0].split()
                    keys = [getrequestkey(i, self.identity) for i in args[0].split()]
                except fcp.FCPProtocolError as e:
                    if len(ids) == 1:
                        print("Cannot retrieve request uri for identity {} - please give a requestkey like {}".format(
                            ids[0], self.seedkeys[0]))
                    else:
                        print("Cannot retrieve request uris for the identities {} - please give requestkeys like {}".format(
                            ids, self.seedkeys[0]))
                    print("Reason: {}".format(e))
                    return
            keys = args[0].split()
            trustifmissing = 0
            commentifmissing = "babcom introduce"

        # store the iterator. If there is at least one captcha in it
        captchaiter = prepareintroduce(ids, keys, self.identity, trustifmissing, commentifmissing)
        try:
            captchas = next(captchaiter)
        except StopIteration:
            pass # iteration finished
        else:
            self.captchaiters.append(captchaiter)
            if captchas is not None:
                return usecaptchas(captchas)

    def do_solvecaptcha(self, *args):
        """Solve a captcha. Usage: solvecaptcha [captcha]"""
        if args and args[0].strip():
            captcha = args[0].strip()
        else:
            if not self.captchas:
                print("no captchas available. Please run introduce.")
                return
            # choose at random from the newest 20 captchas, because
            # pop() after shuffle(l) gave too many repetitions, which
            # seems pretty odd.
            captcha = random.choice(self.captchas[-20:])
            self.captchas.remove(captcha)
        print("Please solve the following CAPTCHA to introduce your identity.")
        try:
            question = captcha.split(" with ")[1]
        except IndexError:
            print("broken CAPTCHA", captcha, "Please run introduce.")
            return
        
        solution = input(question + ": ").strip() # strip away spaces
        while solution == "":
            # catch accidentally hitting enter
            print("Received empty solution. Please type a solution to introduce.")
            solution = input(question + ": ").strip() # strip away spaces
        try:
            captchakey = solvecaptcha(captcha, self.identity, solution)
            print("Inserted own identity to {}".format(captchakey))
        except Exception as e:
            captchakey = _captchasolutiontokey(captcha, solution)
            print("Could not insert identity to {}:\n    {}\n".format(captchakey, e))
            print("Run introduce again to try a different CAPTCHA")

    def do_visibleto(self, *args):
        """Check whether the other can currently see me. Usage: visibleto ID
        Example: visibleto FZynnK5Ngi6yTkBAZXGbdRLHVPvQbd2poW6DmZT8vbs"""
        # TODO: allow using nicknames.
        if args[0] == "":
            print("visibleto needs an ID")
            self.onecmd("help visibleto")
            return
        other = args[0].split()[0]
        # remove name or keypart
        other = identityfrom(other)
        # check whether we’re visible for the otherone
        visible = checkvisible(self.identity, other)
        if visible is None:
            print("We do not know whether", other, "can see you.")
            print("There is no explicit trust but there might be propagating trust.")
            # TODO: check whether I can get the score the other sees for me.
        if visible is False:
            print(other, "marked you as spammer and cannot see anything from you.")
        if visible is True:
            print("You are visible to {}: there is explicit trust.".format(other))
            
    def do_known(self, *args):
        """List all known identities."""
        for i in gettrustees(self.identity):
            name, info = getidentity(i, self.identity)
            if name:
                print(name+"@"+i)

    def do_uploadrecovery(self, *args):
        """Upload recovery information for this identity."""
        print("Uploading recovery information using the recovery secret",
              join_recovery_secret_string(self.recoverysecret0, self.recoverysecret1, self.recoverysecret2))
        upload_recovery(self.recoverysecret0, self.recoverysecret1, self.recoverysecret2,
                        self.identity, self.username,
                        captchasolutions="\n".join(self.captchasolutions))
        print("Please write down the recovery secret:")
        print(join_recovery_secret_string(self.recoverysecret0, self.recoverysecret1, self.recoverysecret2))
        print("Now you can use babcom.py --recover to recover an identity on any computer with the secret.")
        

    def do_exec(self, *args):
        """Execute the code entered. WARNING! Only for development purposes!"""
        code = [args[0]]
        if args[0] == "":
            print("# type your code. Finish with an empty line.")
        nextline = input()
        while nextline != "":
            code.append(nextline)
            nextline = input()
        exec("\n".join(code))
            
    def do_contact(self, *args):
        """Contact someone by private message (Freemail). Usage: contact USK@..."""
        otherkey = args[0]
        otherid = identityfrom(otherkey)
        trustifmissing = 0
        commentifmissing = "Trust received from solving a CAPTCHA" # fake captcha to hide traces. TODO: find a way to set hidden trust 0
        ensureavailability(otherid, otherkey, self.identity,
                           trustifmissing=trustifmissing,
                           commentifmissing=commentifmissing)
        name, info = getidentity(otherid, self.identity)

        with fcp.FCPNode() as n:
            fproxy_port = n.modifyconfig()["current.fproxy.port"]

        send_freemail(self.username + "@" + self.identity,
                      name + "@" + otherid,
                      "test",
                      fproxyport=fproxy_port)
            
    def do_hello(self, *args):
        """Says Hello. Usage: hello [<name>]"""
        name = args[0] if args else 'World'
        print("Hello {}".format(name))

    # TODO: implement do_recover and show the insert URI on start and stop
        
            
    def do_nod(self, *args):
        """News Of the Day. Usage: nod [discover | subscribe | read | share | write | upload <own | share>]"""
        cmd = args[0] if args else 'read'
        if cmd.startswith("write"):
            def nod_input():
                text = []
                nextline = input()
                while nextline != "":
                    text.append(nextline)
                    nextline = input()
                return "\n".join(text)
            if self.nod_own:
                print("You already set the news of the day: {}".format(self.nod_own))
                yn = input("Do you want to replace it? (yes/No) ").strip()
                if not yn.lower().startswith("y"):
                    return
            print("Type your news message. End with an empty line.")
            self.nod_own = nod_input()
            print("New news of the day entry recorded. It will be uploaded tomorrow.")
            print("You can force an irreversible upload with: nod upload own")
            return
        if cmd.startswith("upload"):
            if not cmd.split()[1:]:
                print("Please specify own or share to upload.")
                return self.onecmd("help nod")
            target = cmd.split()[1]
            if target == "own":
                if not self.nod_own:
                    print("No own news of the day set. Use nod write to set it.")
                    return self.onecmd("help nod")
                key = nod_uploadkey(getinsertkey(self.identity), own=True)
                try:
                    fastput(key, self.nod_own)
                except fcp.node.FCPPutFailed as e:
                    print("Could not upload the news of the day. Did you already upload today?")
                    logging.debug(e)
                else:
                    self.nod_own = None
                    print("Uploaded own news of the day. You can upload the next nod tomorrow.")
                return
            if target == "share":
                if not self.nod_shared:
                    print("No shared news of the day set. Use nod share to set it.")
                    return self.onecmd("help nod")
                key = nod_uploadkey(getinsertkey(self.identity), shared=True)
                try:
                    fastput(key, self.nod_shared)
                except fcp.node.FCPPutFailed as e:
                    print("Could not upload the news of the day. Did you already upload today?")
                    logging.debug(e)
                else:
                    self.nod_shared = None
                    print("Uploaded shared news of the day. You can upload the next nod tomorrow.")
                return
        if cmd.startswith("discover"):
            print(getknownids(self.identity, context="babcom"))
            return
            


    def do_quit(self, *args):
        "Leaves the program"
        print("Received quit request. Shutting down!")
        [i.cancel() for i in self.timers]
        self.save()
        if self.spawn:
            self.teardown()
        print("Good bye. Thank you for using babcom!")
        raise SystemExit

    def do_EOF(self, *args):
        "Same as quit. Commonly called via CTRL-D"
        return self.onecmd("quit")

    def emptyline(self, *args):
        "What is done for an empty line"
        print("Type help and hit enter to get help")


class ProtocolError(Exception):
    """
    Did not get the expected reply.
    """
    

def _parse_name(wot_identifier):
    """
    Parse identifier of the forms: nick
                                   nick@key
                                   @key
    :Return: nick, key. If a part is not given return an empty string for it.
    
    >>> _parse_name("BabcomTest@123")
    ('BabcomTest', '123')
    """
    split = wot_identifier.split('@', 1)
    nickname_prefix = split[0]
    key_prefix = (split[1] if split[1:] else '')
    return nickname_prefix, key_prefix


@withprogress
def wotmessage(messagetype, **params):
    """Send a message to the Web of Trust plugin

    >>> name = wotmessage("RandomName")["Replies.Name"]
    """
    params["Message"] = messagetype
    
    def sendmessage(params):
        with fcp.FCPNode() as n:
            return n.fcpPluginMessage(plugin_name="plugins.WebOfTrust.WebOfTrust",
                                      plugin_params=params)[0]
    try:
        resp = sendmessage(params)
    except fcp.FCPProtocolError as e:
        if str(e) == "ProtocolError;No such plugin":
            ensureofficialpluginloaded("WebOfTrust")
            resp = sendmessage(params)
        else: raise
    return resp


def ensureofficialpluginloaded(pluginname):
    """Ensure that the given plugin is loaded.

    :param pluginname: Freemail
    """
    logging.info("Waiting before loading %s until Freenet has at least five connections. Below this it is pointless to try to get an official plugin.", pluginname)
    
    def realpeers(n):
        return [i for i in n.listpeers() if "seed" in i and i["seed"] == "false"]
    with fcp.FCPNode() as n:
        while len(realpeers(n)) < 5:
            time.sleep(1)
    loaded = False
    try:
        with fcp.FCPNode() as n:
            jobid = n._getUniqueId()
            resp = n._submitCmd(jobid, "GetPluginInfo",
                                PluginName=pluginname)[0]
        loaded = True
    except fcp.FCPProtocolError as e:
        if str(e) == "ProtocolError;No such plugin":
            logging.warning("Plugin " + pluginname + " not loaded. Trying to load it.")
            with fcp.FCPNode() as n:
                jobid = n._getUniqueId()
                try:
                    resp = n._submitCmd(jobid, "LoadPlugin",
                                        PluginURL=pluginname,
                                        URLType="official",
                                        OfficialSource="freenet")[0]
                except fcp.node.FCPProtocolError as e:
                    logging.warning("Could not load the Web of Trust: %s", e)
        else: raise
    # if that didn’t wait long enough, just wait longer. longer. Longer. Until it exists.
    retry_timeout_seconds = 300
    start = time.time()
    while not loaded:
        try:
            with fcp.FCPNode() as n:
                jobid = n._getUniqueId()
                resp = n._submitCmd(jobid, "GetPluginInfo",
                                    PluginName=pluginname)[0]
            loaded = True
        except fcp.FCPProtocolError as e:
            logging.warning(str(e))
        if time.time() - start > retry_timeout_seconds:
            # issue another load plugin command.
            return ensureofficialpluginloaded(pluginname)
    logging.info("Plugin Loaded: %s", pluginname)
    return resp



def randomname():
    return wotmessage("RandomName")["Replies.Name"]
        

def createidentity(name="BabcomTest", removedefaultseeds=True, insertkey=None):
    """Create a new Web of Trust identity.

    >>> name = "BabcomTest"
    >>> if slowtests:
    ...     createidentity(name)
    ... else: name
    'BabcomTest'
    
    :returns: the name of the identity created.
    """
    if not name:
        name = wotmessage("RandomName")["Name"]
    if not isinstance(name, str):
        name = name.decode("utf-8")
    msgopts = dict(Nickname=name, Context="babcom", # context cannot be empty
                   PublishTrustList="true", # must use string "true"
                   PublishIntroductionPuzzles="true")
    if insertkey is not None:
        if not isinstance(insertkey, str):
            insertkey = insertkey.decode("utf-8")
        msgopts["InsertURI"] = insertkey
    resp = wotmessage("CreateIdentity", **msgopts)
    if resp['header'] != 'FCPPluginReply' or resp.get('Replies.Message', "") != 'IdentityCreated':
        raise ProtocolError(resp)
    # prune seed-trust, since babcom does its own bootstrapping.
    # TODO: consider changing this when we add support for other services.
    if removedefaultseeds:
        identity = resp['Replies.ID']
        for trustee in gettrustees(identity):
            removetrust(identity, trustee)
    
    return name


def parseownidentitiesresponse(response):
    """Parse the response to Get OwnIdentities from the WoT plugin.

    :returns: [(name, {InsertURI: ..., ...}), ...]

    >>> resp = parseownidentitiesresponse({'Replies.Nickname0': 'FAKE', 'Replies.RequestURI0': 'USK@...', 'Replies.InsertURI0': 'USK@...', 'Replies.Identity0': 'fVzf7fg0Va7vNTZZQNKMCDGo6-FSxzF3PhthcXKRRvA', 'Replies.Message': 'OwnIdentities', 'Success': 'true', 'header': 'FCPPluginReply', 'Replies.Properties0.Property0.Name': 'fake', 'Replies.Properties0.Property0.Value': 'true'})
    >>> list(sorted([(name, list(sorted(info.items()))) for name, info in resp]))
    [('FAKE', [('Contexts', []), ('Identity', 'fVzf7fg0Va7vNTZZQNKMCDGo6-FSxzF3PhthcXKRRvA'), ('InsertURI', 'USK@...'), ('Properties', {'fake': 'true'}), ('RequestURI', 'USK@...'), ('id_num', '0')])]
    """
    field = "Replies.Nickname"
    identities = []
    for i in response:
        if i.startswith(field):
            # format: Replies.Nickname<id_num>
            id_num = i[len(field):]
            nickname = response[i]
            pubkey_hash = response['Replies.Identity{}'.format(id_num)]
            request = response['Replies.RequestURI{}'.format(id_num)]
            insert = response['Replies.InsertURI{}'.format(id_num)]
            contexts = [response[j] for j in response if j.startswith("Replies.Contexts{}.Context".format(id_num))]
            property_keys_keys = [j for j in sorted(response.keys())
                                  if (j.startswith("Replies.Properties{}.Property".format(id_num))
                                      and j.endswith(".Name"))]
            property_value_keys = [j for j in sorted(response.keys())
                                   if (j.startswith("Replies.Properties{}.Property".format(id_num))
                                       and j.endswith(".Value"))]
            properties = dict((response[j], response[k]) for j,k in zip(property_keys_keys, property_value_keys))
            identities.append((nickname, {"id_num": id_num, "Identity":
                                          pubkey_hash, "RequestURI": request, "InsertURI": insert,
                                          "Contexts": contexts, "Properties": properties}))
    return identities


def parseidentityresponse(response):
    """Parse the response to Get OwnIdentities from the WoT plugin.

    :returns: [(name, {InsertURI: ..., ...}), ...]

    >>> resp = {'Replies.Identity': 'jFEicE8bMY0pBN4x6VaN8PsCW342VuuTr0hAc0t39Ls', 'Replies.Identities.0.CurrentEditionFetchState': 'Fetched', 'Replies.Identities.0.Property0.Name': 'IntroductionPuzzleCount', 'Replies.Property0.Value': 10, 'Replies.Rank': 'null', 'Replies.Identities.0.Contexts.0.Name': 'babcom', 'Replies.Identities.0.Property0.Value': 10, 'Replies.Properties0.Property0.Name': 'IntroductionPuzzleCount', 'header': 'FCPPluginReply', 'Replies.Type': 'OwnIdentity', 'Replies.Identities.0.Nickname': 'BabcomTest_other', 'Replies.ID': 'jFEicE8bMY0pBN4x6VaN8PsCW342VuuTr0hAc0t39Ls', 'Replies.Identities.0.Type': 'OwnIdentity', 'Replies.Message': 'Identity', 'Replies.Contexts0.Amount': 2, 'Replies.Identity0': 'jFEicE8bMY0pBN4x6VaN8PsCW342VuuTr0hAc0t39Ls', 'Replies.Identities.Amount': 1, 'Replies.Scores.0.Value': 'Nonexistent', 'Replies.PublishesTrustList0': 'true', 'Replies.RequestURI': 'USK@jFEicE8bMY0pBN4x6VaN8PsCW342VuuTr0hAc0t39Ls,-6yAY9Qq2YildfGRFikIsWQf6RDzPAc84q-gPcbXR7o,AQACAAE/WebOfTrust/1', 'Replies.VersionID': 'a60e2f40-d5e0-4069-8297-05ce8819d817', 'Replies.ID0': 'jFEicE8bMY0pBN4x6VaN8PsCW342VuuTr0hAc0t39Ls', 'Replies.Score0': 'null', 'Replies.CurrentEditionFetchState0': 'Fetched', 'Replies.Contexts0.Context1': 'Introduction', 'Replies.Rank0': 'null', 'Replies.Identities.0.PublishesTrustList': 'true', 'Replies.Contexts.1.Name': 'Introduction', 'Replies.Properties0.Amount': 1, 'Replies.Trust0': 'null', 'Replies.Nickname': 'BabcomTest_other', 'Replies.Identities.0.Properties.0.Value': 10, 'Replies.Properties.0.Value': 10, 'Replies.Score': 'null', 'Replies.Trusts.0.Value': 'Nonexistent', 'Replies.Identities.0.VersionID': '118044f9-0ee8-4986-b798-d0645779ac1b', 'Replies.Properties.0.Name': 'IntroductionPuzzleCount', 'Replies.Context1': 'Introduction', 'Replies.Context0': 'babcom', 'Success': 'true', 'Replies.VersionID0': '84e4aba7-ebfe-4ee6-884f-ea7275decc7b', 'Replies.CurrentEditionFetchState': 'Fetched', 'Replies.Contexts.Amount': 2, 'Replies.Contexts0.Context0': 'babcom', 'Replies.Identities.0.Contexts.1.Name': 'Introduction', 'Replies.Trust': 'null', 'Replies.Properties0.Property0.Value': 10, 'Replies.PublishesTrustList': 'true', 'Identifier': 'id2342652746084203', 'Replies.Nickname0': 'BabcomTest_other', 'Replies.Property0.Name': 'IntroductionPuzzleCount', 'Replies.Contexts.0.Name': 'babcom', 'Replies.Type0': 'OwnIdentity', 'Replies.Properties.Amount': 1, 'Replies.Identities.0.ID': 'jFEicE8bMY0pBN4x6VaN8PsCW342VuuTr0hAc0t39Ls', 'PluginName': 'plugins.WebOfTrust.WebOfTrust', 'Replies.Identities.0.Identity': 'jFEicE8bMY0pBN4x6VaN8PsCW342VuuTr0hAc0t39Ls', 'Replies.Identities.0.RequestURI': 'USK@jFEicE8bMY0pBN4x6VaN8PsCW342VuuTr0hAc0t39Ls,-6yAY9Qq2YildfGRFikIsWQf6RDzPAc84q-gPcbXR7o,AQACAAE/WebOfTrust/1', 'Replies.Identities.0.Context0': 'babcom', 'Replies.Identities.0.Context1': 'Introduction', 'Replies.Identities.0.Properties.0.Name': 'IntroductionPuzzleCount', 'Replies.Identities.0.Contexts.Amount': 2, 'Replies.Identities.0.Properties.Amount': 1, 'Replies.RequestURI0': 'USK@jFEicE8bMY0pBN4x6VaN8PsCW342VuuTr0hAc0t39Ls,-6yAY9Qq2YildfGRFikIsWQf6RDzPAc84q-gPcbXR7o,AQACAAE/WebOfTrust/1'}
    >>> name, info = parseidentityresponse(resp)
    >>> name
    'BabcomTest_other'
    >>> info['RequestURI'].split(",")[-1]
    'AQACAAE/WebOfTrust/1'
    >>> list(sorted(info.keys()))
    ['Contexts', 'CurrentEditionFetchState', 'Identity', 'Properties', 'RequestURI']
    """
    fetchedstate = response["Replies.CurrentEditionFetchState"]
    if fetchedstate != "NotFetched":
        nickname = response["Replies.Nickname"]
    else:
        nickname = None
    pubkey_hash = response['Replies.Identity']
    request = response['Replies.RequestURI']
    contexts = [response[j] for j in response if j.startswith("Replies.Contexts.Context")]
    property_keys_keys = [j for j in sorted(response.keys())
                          if (j.startswith("Replies.Properties")
                              and j.endswith(".Name"))]
    property_value_keys = [j for j in sorted(response.keys())
                           if (j.startswith("Replies.Properties")
                               and j.endswith(".Value"))]
    properties = dict((response[j], response[k]) for j,k in zip(property_keys_keys, property_value_keys))
    info = {"Identity": pubkey_hash, "RequestURI": request,
            "Contexts": contexts, "Properties": properties,
            "CurrentEditionFetchState": fetchedstate}
    return nickname, info


def _requestallownidentities():
    """Get all own identities.

    >>> resp = _requestallownidentities()
    >>> matching = _matchingidentities("BabcomTest", resp)
    >>> # [(name, info) for name, info in matching]
    """
    resp = wotmessage("GetOwnIdentities")
    if resp['header'] != 'FCPPluginReply' or resp.get('Replies.Message', '') != 'OwnIdentities':
        raise ProtocolError(resp)
    return resp

    
def _matchingidentities(prefix, response):
    """Find matching identities in a Web of Trust Plugin response.

    >>> _matchingidentities("BabcomTest", {})
    []
    """
    identities = parseownidentitiesresponse(response)
    nickname_prefix, key_prefix = _parse_name(prefix)
    matches = [(name, info) for name,info in identities
               if (info["Identity"].startswith(key_prefix) and
                   name.startswith(nickname_prefix))]
    # sort the matches by smallest difference to the prefix so that an
    # exact match of the nickname always wins against longer names.
    return sorted(matches, key=lambda match: len(match[0]) - len(nickname_prefix))


def getownidentities(user):
    """Get all own identities which match user."""
    resp = _requestallownidentities()
    return _matchingidentities(user, resp)


def myidentity(user=None):
    """Get an identity from the Web of Trust plugin.

    :param user: Name of the Identity, optionally with additional
                 prefix of the key to disambiguate it.

    If there are multiple IDs matching the name, the user has to
    disambiguate them by selecting one or by adding parts of the
    identity key to the name.

    :returns: [(name, info), ...]
    
    >>> matches = myidentity("BabcomTest")
    >>> matches[0][0]
    'BabcomTest'

    """
    if user is None:
        user = createidentity()
    else:
        if not in_doctest:
              print("... checking for existing identities which match", user, "...")
    matches = getownidentities(user)
    if not matches:
        if not in_doctest:
              print("... no matches found, creating new identity...")
        createidentity(user)
        matches = getownidentities(user)
        
    return matches


def getidentity(identity, truster):
    """Get all own identities which match user.

    >>> othername = "BabcomTest_other"
    >>> if slowtests:
    ...     matches = myidentity("BabcomTest")
    ...     name, info = matches[0]
    ...     truster = info["Identity"]
    ...     matches = myidentity(othername)
    ...     name, info = matches[0]
    ...     identity = info["Identity"]
    ...     name, info = getidentity(identity, truster)
    ...     name
    ... else: othername
    'BabcomTest_other'
    """
    resp = wotmessage("GetIdentity",
                      Identity=identity, Truster=truster)
    if resp['header'] != 'FCPPluginReply' or resp.get('Replies.Message', '') != 'Identity':
        raise ProtocolError(resp)

    name, info = parseidentityresponse(resp)
    return name, info


def addcontext(identity, context):
    """Add a context to an identity to show others that it supports a certain service.

    >>> matches = myidentity("BabcomTest")
    >>> name, info = matches[0]
    >>> identity = info["Identity"]
    >>> addcontext(identity, "testadd")
    >>> matches = myidentity(name)
    >>> info = matches[0][1]
    >>> "testadd" in info["Contexts"]
    True
    """
    resp = wotmessage("AddContext",
                      Identity=identity,
                      Context=context)
    if resp['header'] != 'FCPPluginReply' or resp.get('Replies.Message', '') != 'ContextAdded':
        raise ProtocolError(resp)
    

def removecontext(identity, context):
    """Add a context to an identity to show others that it supports a certain service.

    >>> matches = myidentity("BabcomTest")
    >>> name, info = matches[0]
    >>> identity = info["Identity"]
    >>> addcontext(identity, "testremove")
    >>> removecontext(identity, "testremove")
    >>> removecontext(identity, "testadd")
    """
    resp = wotmessage("RemoveContext",
                      Identity=identity,
                      Context=context)
    if resp['header'] != 'FCPPluginReply' or resp.get('Replies.Message', '') != 'ContextRemoved':
        raise ProtocolError(resp)
    

def ssktousk(ssk, foldername):
    """Convert an SSK to a USK.

    >>> ssktousk("SSK@pAOgyTDft8bipMTWwoHk1hJ1lhWDvHP3SILOtD1e444,Wpx6ypjoFrsy6sC9k6BVqw-qVu8fgyXmxikGM4Fygzw,AQACAAE/", "folder")
    'USK@pAOgyTDft8bipMTWwoHk1hJ1lhWDvHP3SILOtD1e444,Wpx6ypjoFrsy6sC9k6BVqw-qVu8fgyXmxikGM4Fygzw,AQACAAE/folder/0'
    """
    return "".join(("U", ssk[1:].split("/")[0],
                    "/", foldername, "/0"))


def usktossk(usk, pathname):
    """Convert a USK to an SSK with the given pathname.

    >>> usktossk("USK@pAOgyTDft8bipMTWwoHk1hJ1lhWDvHP3SILOtD1e444,Wpx6ypjoFrsy6sC9k6BVqw-qVu8fgyXmxikGM4Fygzw,AQACAAE/", "folder")
    'SSK@pAOgyTDft8bipMTWwoHk1hJ1lhWDvHP3SILOtD1e444,Wpx6ypjoFrsy6sC9k6BVqw-qVu8fgyXmxikGM4Fygzw,AQACAAE/folder'
    """
    return "".join(("S" + usk[1:].split("/")[0],
                    "/", pathname))


@withprogress
def fastput(private, data, node=None):
    """Upload a small amount of data as fast as possible.

    :param data: a string of bytes. Take care to encode strings to utf-8!

    >>> with fcp.FCPNode() as n:
    ...    pub, priv = n.genkey(name="hello.txt")
    >>> if slowtests:
    ...     pubtoo = fastput(priv, b"Hello Friend!")
    >>> with fcp.FCPNode() as n:
    ...    pub, priv = n.genkey()
    ...    insertusk = ssktousk(priv, "folder")
    ...    data = b"Hello USK"
    ...    if slowtests:
    ...        pub = fastput(insertusk, data, node=n)
    ...        dat = fastget(pub)[1]
    ...    else:
    ...        pub = "something,AQACAAE/folder/0"
    ...        dat = data
    ...    pub.split(",")[-1], dat.decode("utf-8")
    ('AQACAAE/folder/0', 'Hello USK')
    """
    def n():
        if node is None or node.running == False:
            return fcp.FCPNode()
        return node
    # ensure that we deal in string
    if (PY3 and not isinstance(private, str)):
        private = private.decode("utf-8")
    # and in bytes for the data
    if (PY3 and isinstance(data, str)):
        data = data.encode("utf-8")
    with n() as node:
        return node.put(uri=private, data=data,
                        mimetype="application/octet-stream",
                        realtime=True, priority=1)


@withprogress
def fastget(public, node=None):
    """Download a small amount of data as fast as possible.

    :param public: the (public) key of the data to fetch.

    :returns: the data the key references as bytes (decode to utf-8 to get a string).

    On failure it raises an Exception.

    Note: only use this for small files. For large files it is slower
    than regular node.get() and might block other usage of the node.

    >>> with fcp.FCPNode() as n:
    ...    pub, priv = n.genkey(name="hello.txt")
    ...    data = b"Hello Friend!"
    ...    if slowtests:
    ...        pubkey = fastput(priv, data, node=n)
    ...        fastget(pub, node=n)[1].decode("utf-8")
    ...    else: data.decode("utf-8")
    'Hello Friend!'

    """
    def n():
        if node is None or node.running == False:
            return fcp.FCPNode()
        return node
    # ensure that we deal in string
    if (PY3 and not isinstance(public, str)):
        public = public.decode("utf-8")
    with n() as node:
        return node.get(public,
                        realtime=True, priority=1,
                        followRedirect=True)


def getinsertkey(identity):
    """Get the insert key of the given identity.

    >>> matches = myidentity("BabcomTest")
    >>> name, info = matches[0]
    >>> identity = info["Identity"]
    >>> insertkey = getinsertkey(identity)
    >>> insertkey.split("/")[0].split(",")[-1]
    'AQECAAE'
    """
    resp = _requestallownidentities()
    identities = parseownidentitiesresponse(resp)
    insertkeys = [info["InsertURI"]
                  for name,info in identities
                  if info["Identity"] == identity]
    if insertkeys[1:]:
        raise ProtocolError(
            "More than one insert key for the same identity: {}".format(
                insertkeys))
    return insertkeys[0]


def getrequestkey(identity, truster):
    """Get the request key of the given identity.

    >>> matches = myidentity("BabcomTest")
    >>> name, info = matches[0]
    >>> identity = info["Identity"]
    >>> requestkey = getrequestkey(identity, identity)
    >>> requestkey.split("/")[0].split(",")[-1]
    'AQACAAE'
    """
    name, info = getidentity(identity, truster)
    requestkey = info["RequestURI"]
    return requestkey


def identityfrom(identitykey):
    """Get the identity for the key.

    :param identitykey: name@identity, insertkey or requestkey: USK@...,...,AQ.CAAE/WebOfTrust/N
    
    >>> identityfrom("USK@pAOgyTDft8bipMTWwoHk1hJ1lhWDvHP3SILOtD1e444,Wpx6ypjoFrsy6sC9k6BVqw-qVu8fgyXmxikGM4Fygzw,AQACAAE/WebOfTrust/0")
    'pAOgyTDft8bipMTWwoHk1hJ1lhWDvHP3SILOtD1e444'
    """
    if "@" in identitykey:
        identitykey = identitykey.split("@")[1]
    if "/" in identitykey:
        identitykey = identitykey.split("/")[0]
    if "," in identitykey:
        identitykey = identitykey.split(",")[0]
    return identitykey


def createcaptchas(number=20, seed=None):
    """Create text captchas

    >>> createcaptchas(number=1, seed=42)
    [('KSK@sJBz_USRK_zHvz_? with 38 plus 28 = ?', 'sJBz_USRK_zHvz_66')]
    
    :returns: [(captchatext, solution), ...]
    """
    # prepare the random number generator for reproducible tests.
    random.seed(seed)
    
    def plus(x, y):
        "KSK@{2}? with {0} plus {1} = ?"
        return x + y
        
    def minus(x, y):
        "KSK@{2}? with {0} minus {1} = ?"
        return x - y
        
    def plusequals(x, y):
        "KSK@{2}? with {0} plus ? = {1}"
        return y - x
        
    def minusequals(x, y):
        "KSK@{2}? with {0} minus ? = {1}"
        return x + y
        
    questions = [plus, minus,
                 plusequals,
                 minusequals]

    captchas = []
    
    def fourletters():
        return [random.choice("ABCDEFHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")
                for i in range(4)]
    
    def secret():
        return "".join(fourletters() + ["_"] +
                       fourletters() + ["_"] +
                       fourletters() + ["_"])
    
    for i in range(number):
        sec = secret()
        question = random.choice(questions)
        x = random.randint(1, 49)
        y = random.randint(1, 49)
        captcha = question.__doc__.format(x, y, sec)
        solution = sec + str(question(x, y))
        captchas.append((captcha, solution))

    return captchas


def getcaptchausk(identitykey):
    """Turn a regular identity key (request or insert) into a captcha key.

    >>> fromssk = getcaptchausk("SSK@pAOgyTDft8bipMTWwoHk1hJ1lhWDvHP3SILOtD1e444,Wpx6ypjoFrsy6sC9k6BVqw-qVu8fgyXmxikGM4Fygzw,AQACAAE/")
    >>> fromusk = getcaptchausk("USK@pAOgyTDft8bipMTWwoHk1hJ1lhWDvHP3SILOtD1e444,Wpx6ypjoFrsy6sC9k6BVqw-qVu8fgyXmxikGM4Fygzw,AQACAAE/WebOfTrust/0")
    >>> fromrawssk = getcaptchausk("SSK@pAOgyTDft8bipMTWwoHk1hJ1lhWDvHP3SILOtD1e444,Wpx6ypjoFrsy6sC9k6BVqw-qVu8fgyXmxikGM4Fygzw,AQACAAE")
    >>> fromsskfile = getcaptchausk("SSK@pAOgyTDft8bipMTWwoHk1hJ1lhWDvHP3SILOtD1e444,Wpx6ypjoFrsy6sC9k6BVqw-qVu8fgyXmxikGM4Fygzw,AQACAAE/file.txt")
    >>> fromssk == fromusk == fromrawssk == fromsskfile
    True
    >>> fromssk
    'USK@pAOgyTDft8bipMTWwoHk1hJ1lhWDvHP3SILOtD1e444,Wpx6ypjoFrsy6sC9k6BVqw-qVu8fgyXmxikGM4Fygzw,AQACAAE/babcomcaptchas/0'
    """
    rawkey = identitykey.split("/")[0]
    ssk = "S" + rawkey[1:]
    return ssktousk(ssk, "babcomcaptchas")
    

def insertcaptchas(identity):
    """Insert a list of CAPTCHAs.

    >>> matches = myidentity("BabcomTest")
    >>> name, info = matches[0]
    >>> identity = info["Identity"]
    >>> if slowtests:
    ...     usk, solutions = insertcaptchas(identity)
    ...     solutions[0][:4]
    ... else: "KSK@"
    'KSK@'

    :returns: captchasuri, ["KSK@solution", ...]
    """
    insertkey = getinsertkey(identity)
    captchas = createcaptchas()
    captchasdata = "\n".join(captcha for captcha,solution in captchas)
    captchasolutions = [solution for captcha,solution in captchas]
    captchausk = getcaptchausk(insertkey)
    with fcp.FCPNode() as n:
        pub = fastput(captchausk, captchasdata.encode("utf-8"), node=n)
    return pub, ["KSK@" + solution
                 for solution in captchasolutions]
    

def providecaptchas(identity):
    """Provide a link to the CAPTCHA queue as property of the identity.

    >>> matches = myidentity("BabcomTest")
    >>> name, info = matches[0]
    >>> identity = info["Identity"]
    >>> if slowtests:
    ...     solutions = providecaptchas(identity)
    ...     matches = myidentity("BabcomTest")
    ...     name, info = matches[0]
    ...     "babcomcaptchas" in info["Properties"]
    ... else: True
    True
    
    :returns: ["KSK@...", ...] # the solutions to watch
    """
    pubusk, solutions = insertcaptchas(identity)
    resp = wotmessage("SetProperty", Identity=identity,
                      Property="babcomcaptchas",
                      Value=pubusk)
    if resp['header'] != 'FCPPluginReply' or resp.get('Replies.Message', "") != 'PropertyAdded':
        raise ProtocolError(resp)

    return solutions


def _captchasolutiontokey(captcha, solution):
    """Turn the CAPTCHA and its solution into a key.
    
    >>> captcha = 'KSK@hBQM_njuE_XBMb_? with 10 plus 32 = ?'
    >>> solution = '42'
    >>> _captchasolutiontokey(captcha, solution)
    b'KSK@hBQM_njuE_XBMb_42'
    """
    secret = captcha.split("?")[0]
    key = secret + str(solution)
    return key.encode("utf-8")
    

def solvecaptcha(captcha, identity, solution):
    """Use the solution to solve the CAPTCHA.

    >>> captcha = 'KSK@hBQM_njuE_XBMl_? with 10 plus 32 = ?'
    >>> solution = str(int(time.time()*1000)) + str(random.randint(1, 42))
    >>> matches = myidentity("BabcomTest")
    >>> name, info = matches[0]
    >>> identity = info["Identity"]
    >>> idrequestkey = getrequestkey(identity, identity)
    >>> if slowtests:
    ...     captchakey = solvecaptcha(captcha, identity, solution)
    ...     idrequestkey == fastget(captchakey)[1].decode("utf-8")
    ... else: True
    True
    """
    captchakey = _captchasolutiontokey(captcha, solution)
    idkey = getrequestkey(identity, identity)
    return fastput(captchakey, idkey)


def gettrust(truster, trustee):
    """Set trust to an identity.

    >>> my = myidentity("BabcomTest")[0][1]["Identity"]
    >>> other = myidentity("BabcomTest_other")[0][1]["Identity"]
    >>> gettrust(my, other)
    'Nonexistent'
    """
    resp = wotmessage("GetTrust",
                      Truster=truster, Trustee=trustee)
    if resp['header'] != 'FCPPluginReply' or resp.get('Replies.Message', "") != 'Trust':
        raise ProtocolError(resp)
    return resp['Replies.Trusts.0.Value']


def settrust(myidentity, otheridentity, trust, comment):
    """Set trust to an identity.

    :param trust: -100..100. 
                  -100 to -2: report as spammer, do not download.
                  -1: do not download.
                   0: download and show.
                   1 to 100: download, show and mark as non-spammer so
                       others download the identity, too.
    """
    resp = wotmessage("SetTrust",
                      Truster=myidentity, Trustee=otheridentity,
                      Value=str(trust), Comment=comment)
    if resp['header'] != 'FCPPluginReply' or resp.get('Replies.Message', "") != 'TrustSet':
        raise ProtocolError(resp)


def removetrust(myidentity, otheridentity):
    """Remove the trust of an identity."""
    resp = wotmessage("RemoveTrust",
                      Truster=myidentity, Trustee=otheridentity)
    if resp['header'] != 'FCPPluginReply' or resp.get('Replies.Message', "") != 'TrustRemoved':
        raise ProtocolError(resp)


def addidentity(requesturi):
    """Ensure that WoT knows the given identity."""
    resp = wotmessage("AddIdentity",
                      RequestURI=requesturi)
    if resp['header'] != 'FCPPluginReply' or resp.get('Replies.Message', "") != 'IdentityAdded':
        raise ProtocolError(resp)


def watchcaptchas(solutions):
    """Watch the solutions to the CAPTCHAs
    
    :param solutions: Freenet Keys where others can upload solved CAPTCHAs. 

    :returns: generator which yields None or (key, data).
              <generator with ('KSK@...<captchakey>', 'USK@...<identity>')...>

    # TODO: check whether returning (None, None) or (key, data) 
            would lead to better code.

    Just call watcher = watchcaptchas(solutions), then you can ask
    watcher whether there’s a solution via watcher.next(). It should
    return after roughly 10ms, either with None or with (key, data)
    
    >>> d1 = b"Test"
    >>> d2 = b"Test2"
    >>> k1 = "KSK@tcshrietshcrietsnhcrie-Test1"
    >>> k2 = "KSK@tcshrietshcrietsnhcrie-Test2"
    >>> if slowtests:
    ...     k1res = fastput(k1, d1)
    ...     k2res = fastput(k2, d2)
    ...     watcher = watchcaptchas([k1,k2])
    ...     list(sorted([i for i in watcher if i is not None])) # drain watcher.
    ...     # note: I cannot use i.next() in the doctest, else I’d get "I/O operation on closed file"
    ... else:
    ...     [(k1, bytearray(d1)), (k2, bytearray(d2))]
    [('KSK@tcshrietshcrietsnhcrie-Test1', bytearray(b'Test')), ('KSK@tcshrietshcrietsnhcrie-Test2', bytearray(b'Test2'))]

    """
    # TODO: in Python3 this could be replaced with less than half the lines using futures.
    # use a shared fcp connection for all get requests
    node = fcp.FCPNode()
    tasks = []
    for i in solutions:
        job = node.get(i,
                       realtime=True, priority=4,
                       followRedirect=False,
                       **{"async": True})
        tasks.append((i, job))
    
    while tasks:
        atleastone = False
        for key, job in tasks:
            if job.isComplete():
                tasks.remove((key, job))
                atleastone = True
                res = key, job.getResult()[1]
                yield res
        if not atleastone:
            yield None # no job finished
    
    # close the node. Use a new one for the next run.
    node.shutdown()


def ensureavailability(identity, requesturi, ownidentity, trustifmissing, commentifmissing):
    """Ensure that the given identity is available in the WoT, adding trust as necessary.
    
    :returns: True if trust had to be added, else False
    """
    try:
        name, info = getidentity(identity, ownidentity)
    except ProtocolError as e:
        unknowniderror = 'plugins.WebOfTrust.exceptions.UnknownIdentityException: {}'.format(identity)
        if e.args[0]['Replies.Description'] == unknowniderror:
            logging.warning("identity {} not yet known. Adding trust {}".format(identity, trustifmissing))
            addidentity(requesturi)
            settrust(ownidentity, identity, trustifmissing, commentifmissing)
            return True
        else:
            raise
    return False
                    
                
def prepareintroduce(identities, requesturis, ownidentity, trustifmissing, commentifmissing):
    """Prepare announcing to the identities.

    This ensures that the identity is known to WoT, gives it trust to
    ensure that it will be fetched, pre-fetches the ID and fetches the
    captchas. It returns an iterator which yields either a captcha to
    solve or None.
    """
    # ensure that we have a real copy to avoid mutating the original lists.
    ids = identities[:]
    keys = requesturis[:]
    tasks = list(zip(ids, keys))
    # use a single node for all the the get requests in the iterator.
    node = fcp.FCPNode()
    while tasks:
        for identity, requesturi in tasks[:]:
            ensureavailability(identity, requesturi, ownidentity, trustifmissing, commentifmissing)
            try:
                print("Getting identity information for {}".format(identity))
                name, info = getidentity(identity, ownidentity)
            except ProtocolError as e:
                unknowniderror = 'plugins.WebOfTrust.exceptions.UnknownIdentityException: {}'.format(identity)
                if e.args[0]['Replies.Description'] == unknowniderror:
                    logging.warning("identity to introduce not yet known. Adding trust {} for {}".format(trustifmissing, identity))
                    addidentity(requesturi)
                    settrust(ownidentity, identity, trustifmissing, commentifmissing)
                name, info = getidentity(identity, ownidentity)
            if "babcomcaptchas" in info["Properties"]:
                print("Getting CAPTCHAs for id", identity)
                captchas = fastget(info["Properties"]["babcomcaptchas"],
                                   node=node)[1].decode("utf-8")
                # task finished
                tasks.remove((identity, requesturi))
                yield captchas
            else:
                if info["CurrentEditionFetchState"] == "NotFetched":
                    print("Cannot introduce to identity {}, because it has not been fetched, yet.".format(identity))
                    trust = gettrust(ownidentity, identity)
                    if trust == "Nonexistent" or int(trust) >= 0:
                        if trust == "Nonexistent":
                            print("No trust set yet. Setting trust", trustifmissing, "to ensure that identity {} gets fetched.".format(identity))
                            settrust(ownidentity, identity, trustifmissing, commentifmissing)
                        else:
                            print("The identity has trust {}, so it should be fetched soon.".format(trust))
                        print("firing get({}) in background to make it more likely that the ID is fetched quickly (since it’s already in the local store, then).".format(requesturi))
                        node.get(requesturi, followRedirect=True,
                                 persistence="reboot", nodata=True, **{"async": True})
                        # use the captchas without going through Web of Trust to avoid a slowpath
                        print("Getting the captchas from {}".format(requesturi))
                        captchas = fastget(getcaptchausk(requesturi),
                                           node=node)[1].decode("utf-8")
                        # task finished
                        tasks.remove((identity, requesturi))
                        yield captchas
                    else:
                        print("You marked this identity as spammer or disruptive by setting trust {}, so it cannot be fetched.".format(trust))
                        # task finished: it cannot be done
                        tasks.remove((identity, requesturi))
                else:
                    name, info = getidentity(identity, ownidentity)
                    # try to go around WoT
                    captchausk = getcaptchausk(info["RequestURI"])
                    try:
                        yield fastget(captchausk,
                                      node=node)[1].decode("utf-8")
                    except Exception as e:
                        print("Identity {}@{} published no CAPTCHAs, cannot introduce to it.".format(name, identity))
                        print("reason:", e)
                    tasks.remove((identity, requesturi))
    # close the FCP connection when all tasks are done.
    node.shutdown()


def parsetrusteesresponse(response):
    """Parse the response to GetTrustees from the WoT plugin.

    :returns: [(name, {InsertURI: ..., ...}), ...]

    >>> resp = parseownidentitiesresponse({'Replies.Nickname0': 'FAKE', 'Replies.RequestURI0': 'USK@...', 'Replies.InsertURI0': 'USK@...', 'Replies.Comment0': 'Fake', 'Replies.Identity0': 'fVzf7fg0Va7vNTZZQNKMCDGo6-FSxzF3PhthcXKRRvA', 'Replies.Message': 'dentities', 'Success': 'true', 'header': 'FCPPluginReply', 'Replies.Properties0.Property0.Name': 'fake', 'Replies.Properties0.Property0.Value': 'true'})
    >>> [(name, list(sorted(info.items()))) for name, info in resp]
    [('FAKE', [('Contexts', []), ('Identity', 'fVzf7fg0Va7vNTZZQNKMCDGo6-FSxzF3PhthcXKRRvA'), ('InsertURI', 'USK@...'), ('Properties', {'fake': 'true'}), ('RequestURI', 'USK@...'), ('id_num', '0')])]
    """
    field = "Replies.Identity"
    identities = []
    for i in response:
        if i.startswith(field):
            # format: Replies.Nickname<id_num>
            id_num = i[len(field):]
            pubkey_hash = response['Replies.Identity{}'.format(id_num)]
            request = response['Replies.RequestURI{}'.format(id_num)]
            nickname = response.get("Replies.Nickname{}".format(id_num), None)
            comment = response.get("Replies.Comment{}".format(id_num), None)
            contexts = [response[j] for j in response if j.startswith("Replies.Contexts{}.Context".format(id_num))]
            property_keys_keys = [j for j in sorted(response.keys())
                                  if (j.startswith("Replies.Properties{}.Property".format(id_num))
                                      and j.endswith(".Name"))]
            property_value_keys = [j for j in sorted(response.keys())
                                   if (j.startswith("Replies.Properties{}.Property".format(id_num))
                                       and j.endswith(".Value"))]
            properties = dict((response[j], response[k]) for j,k in zip(property_keys_keys, property_value_keys))
            identities.append((pubkey_hash, {"id_num": id_num, "Comment": comment, "Nickname":
                                             nickname, "RequestURI": request,
                                             "Contexts": contexts, "Properties": properties}))
    return identities


def gettrustees(identity, context=""):
    resp = wotmessage("GetTrustees", Identity=identity,
                      Context=context) # "" means any context
    return dict(parsetrusteesresponse(resp))

                    
def getknownids(identity, context=""):
    resp = wotmessage("GetIdentitiesByScore", Truster=identity,
                      Selection="+",
                      Context=context) # "" means any context
    # TODO: parse the IDs by score response
    return resp

                    
def checkvisible(ownidentity, otheridentity):
    """Check whether the other identity can see me."""
    # TODO: get the exact trust value
    trustees = gettrustees(otheridentity)
    if ownidentity in trustees:
        return True


def recovery_secret_to_ksk(recovery_secret):
    """Turn secret and salt to the URI.
    
    >>> recovery_secret_to_ksk("0123.4567!ABCD")
    'KSK@babcom-recovery-0123.4567!ABCD'
    """
    if isinstance(recovery_secret, bytes):
        recovery_secret = recovery_secret.decode("utf-8")
    # fix possible misspellings (the recovery_secret is robust against them)
    for err, fix in ["l1", "I1", "O0", "_-", "*+"]:
        recovery_secret = recovery_secret.replace(err, fix)
    ksk = "KSK@babcom-recovery-" + recovery_secret
    return ksk


def recovery_secret_to_usk(recovery_secret, key):
    """Turn secret and salt to the URI. This is a DUMB idea, because
    anyone who knows the key can try to attack files stored on disk.

    """
    if isinstance(recovery_secret, bytes):
        recovery_secret = recovery_secret.decode("utf-8")
    if isinstance(key, bytes):
        key = key.decode("utf-8")
    # fix possible misspellings (the recovery_secret is robust against them)
    for err, fix in ["l1", "I1", "O0", "_-", "*+"]:
        recovery_secret = recovery_secret.replace(err, fix)
    usk = "U" + key[1:].split("/")[0] + "/" + recovery_secret
    return usk


def create_recovery_secret_part(nblocks=3):
    """Create a random password for recovering an identity.

    >>> secret = create_recovery_secret_part(3)
    >>> len(secret)
    14
    """
    letters = "0123456789ABCDEFGHJKLMNPQRSTUVWXabcdefghijkmnopqrstuvwx"
    delimiters = "-=.+"
    pw = ""
    for i in range(nblocks*4):
        if i % 4 == 0 and i != 0 and i != nblocks*4:
            pw += random.choice(delimiters)
        pw += random.choice(letters)
    return pw


def split_recovery_secret_string(secret):
    return secret.split("/")


def join_recovery_secret_string(*parts):
    return "/".join(parts)


def salt_and_iterate_recovery_secret(secret, salt):
    if isinstance(salt, str):
        salt = salt.encode("utf-8")
    if isinstance(secret, str):
        secret = secret.encode("utf-8")
    if isinstance(salt, bytearray):
        salt = bytes(salt)
    if isinstance(secret, bytearray):
        secret = bytes(secret)
    # FIXME: actually hash plus iterate
    logging.debug("secret: %s, salt: %s", secret, salt)
    return passlib_hash.pbkdf2_sha256.using(
        salt=salt, rounds=20000).hash( # 20000 rounds take about 1 second
            secret).replace("/", "-")


@withprogress
def upload_recovery(recovery_date, recovery_secret1, recovery_secret2, ownidentity, username, **kwds):
    """Upload recovery information for the ownidentity."""
    insertkey = getinsertkey(ownidentity)
    secret1 = recovery_date + "--" + recovery_secret1
    uploadprefix = recovery_secret_to_ksk(secret1)
    if isinstance(username, str):
        username = username.encode("utf-8")
    if isinstance(ownidentity, str):
        ownidentity = ownidentity.encode("utf-8")
    nameandid = username + b"@" + ownidentity
    nameandid = bytes(nameandid)
    logging.debug("%s: %s", "nameandid", nameandid)
    salted = salt_and_iterate_recovery_secret(recovery_secret2, nameandid)
    uploadprefix_secure = recovery_secret_to_ksk(salted)
    keys = []
    toupload = [(uploadprefix + "--identity", nameandid),
                (uploadprefix_secure + "--insertkey", insertkey),
                (uploadprefix_secure + "--metainfo", "\n".join(kwds.keys()))]
    meta = [(uploadprefix_secure + "--" + k, v) for k, v in kwds.items()]
    toupload.extend(meta)
    with fcp.FCPNode() as n:
        tasks = []
        for uri, data in toupload:
            job = n.put(uri=uri, data=data,
                        realtime=True, priority=1,
                        mimetype="application/octet-stream",
                        **{"async": True})
            tasks.append((uri, job))
        for uri, job in tasks:
            while not job.isComplete():
                time.sleep(0.2)
            keys.append(job.getResult()[1])
    return keys
    

def recover_request_key_and_name(recovery_date, recovery_secret1):
    """Download the private information about this ID from Freenet."""
    secret1 = recovery_date + "--" + recovery_secret1
    uploadprefix = recovery_secret_to_ksk(secret1)
    return fastget(uploadprefix + "--identity")[1]


def recover_insert_key(recovery_secret2, ownidentity, username):
    """Download the private information about this ID from Freenet."""
    nameandid = username + b"@" + ownidentity
    nameandid = bytes(nameandid)
    logging.warning("%s: %s", "nameandid", nameandid)
    salted = salt_and_iterate_recovery_secret(recovery_secret2, nameandid)
    uploadprefix = recovery_secret_to_ksk(salted)
    return fastget(uploadprefix + "--insertkey")[1]


# The Freemail functionality is adapted from the Infocalypse work of Steve Dougherty

def send_freemail(from_nickidentity, to_nickidentity, message,
                  password="12345", # FIXME: Use a proper password
                  mailhost="127.0.0.1", smtpport=4025,
                  fproxyhost="127.0.0.1", fproxyport="8888"):
    """
    Prompt for a pull request message, and send a pull request from
    from_identity to to_identity for the repository to_repo_name.

    :type to_nickidentity: USER@id
    :type from_nickidentity: USER@id
    """
    setup_freemail(from_nickidentity,
                   mailhost, smtpport,
                   fproxyhost, fproxyport)
    from_address = require_freemail(from_nickidentity)
    to_address = require_freemail(to_nickidentity)

    # TODO: Will there always be a request URI set in the config? What about
    # a path? The repo could be missing a request URI, if that URI is
    # set manually. We could check whether the default path is a
    # freenet path. We cannot be sure whether the request uri will
    # always be the uri we want to send the pull-request to, though:
    # It might be an URI we used to get some changes which we now want
    # to send back to the maintainer of the canonical repo.

    # TODO: Save message and load later in case sending fails.

    source_lines = message.splitlines()

    source_lines = [line for line in source_lines]

    # Body is third line and after.
    msg = MIMEText('\n'.join(source_lines[2:]))
    msg['Subject'] = "[babcom] " + source_lines[0]
    msg['To'] = to_address
    msg['From'] = from_address

    host = mailhost
    port = smtpport
    smtp = smtplib.SMTP(host, port)
    try:
        smtp.login(from_address, password)
    except smtplib.SMTPAuthenticationError:
        logging.error("Could not log in with password %s and email %s", password, from_address)
        logging.error("Message should have been: \n%s", source_lines)
        return
    smtp.sendmail(from_address, to_address, msg.as_string())


def require_freemail(identity_with_nickname):
    """
    Return the given identity's Freemail address.
    Abort with an error message if the given identity does not have a
    Freemail address / context.

    :param identity_with_nickname: <user>@<identity>
    """
    nickname, identity = _parse_name(identity_with_nickname)
    re_encode = base64.b32encode(fcp.node.base64decode(identity))
    # Remove trailing '=' padding.
    re_encode = re_encode.decode("utf-8").rstrip('=')
    
    # Freemail addresses are lower case.
    address = nickname + '@' + re_encode  + '.freemail'
    return address.lower()


def setup_freemail(local_id, mailhost, smtpport, fproxyhost, fproxyport):
    """
    Test, and set a Freemail password for the identity.

    :returns: password
    """
    ensureofficialpluginloaded("Freemail_wot")
    
    address = require_freemail(local_id)

    # FIXME: use a proper password
    password = "12345"

    host = mailhost
    port = smtpport
    
    # Check that the password works.
    try:
        # TODO: Is this the correct way to get the configured host?
        smtp = smtplib.SMTP(host, port)
        smtp.login(address, password)
    except smtplib.SMTPAuthenticationError as e:
        print("Could not log in with the given password.\nGot '{0}'\n".format(e.smtp_error))
        print("currently you need to visit", "http://" + fproxyhost + ":" + str(fproxyport) + "/Freemail", "and create an account with password", password)
        return
    except smtplib.SMTPConnectError as e:
        print("Could not connect to server.\nGot '{0}'\n".format(e.smtp_error))
        return

    return password


def check_freemail(local_identity, password="12345", # FIXME: use a proper password
                   mailhost="127.0.0.1", imapport=4143):
    """
    Check Freemail for local_identity and print information on any VCS
    messages received.

    :type local_identity: Local_WoT_ID
    """
    address = require_freemail(local_identity)

    # Log in and open inbox.
    host = mailhost
    port = imapport
    imap = imaplib.IMAP4(host, port)
    imap.login(address, password)
    imap.select()

    # Parenthesis to work around erroneous quotes:
    # http://bugs.python.org/issue917120
    reply_type, message_numbers = imap.search(None, '(SUBJECT %s)' % "[babcom]")

    # imaplib returns numbers in a singleton string separated by whitespace.
    message_numbers = message_numbers[0].split()

    if not message_numbers:
        # TODO: Is aborting appropriate here? Should this be ui.status and
        # return?
        print("No messages found.")
        return

    # fetch() expects strings for both. Individual message numbers are
    # separated by commas. It seems desirable to peek because it's not yet
    # apparent that this is a [vcs] message with YAML.
    # Parenthesis to prevent quotes: http://bugs.python.org/issue917120
    status, subjects = imap.fetch(','.join(message_numbers),
                                  r'(body[header.fields Subject])')

    # Expecting 2 list items from imaplib for each message, for example:
    # ('5 (body[HEADER.FIELDS Subject] {47}', 'Subject: [vcs]  ...\r\n\r\n'),
    # ')',

    # Exclude closing parens, which are of length one.
    subjects = [x for x in subjects if len(x) == 2]

    subjects = [x[1] for x in subjects]

    # Match message numbers with subjects; remove prefix and trim whitespace.
    subjects = dict((message_number, subject[len('Subject: '):].rstrip()) for
                    message_number, subject in zip(message_numbers, subjects))

    for message_number, subject in subjects.items():
        status, fetched = imap.fetch(str(message_number),
                                     r'(body[text] '
                                     r'body[header.fields From)')

        # Expecting 3 list items, as with the subject fetch above.
        body = fetched[0][1]
        from_address = fetched[1][1][len('From: '):].rstrip()

        yield from_address, subject, body


@withprogress
def wait_until_online(fcp_port):
    return freenet.spawn.wait_until_online(fcp_port)


def nod_uploadkey(private, own=False, date=None):
    """Generate the key for the next News of the Day entry.

    >>> nod_uploadkey("USK@foo,moo,goo/WebOfTrust/0", own=False, date=datetime.datetime(2010,1,1))
    'SSK@foo,moo,goo/nod-shared-2010-01-01'
    >>> nod_uploadkey("USK@foo,moo,goo/WebOfTrust/0", own=True, date=datetime.datetime(2010,2,1))
    'SSK@foo,moo,goo/nod-own-2010-02-01'
    """
    if date:
        t = date
    else:
        t = today = datetime.datetime(*time.gmtime()[:6])
    datepart = "{:04}-{:02}-{:02}".format(t.year, t.month, t.day)
    if own:
        path = "nod-own-" + datepart
    else:
        path = "nod-shared-" + datepart
    return usktossk(private, path)



def _test(verbose=None):

    """Run the tests

    >>> True
    True
    """
    import doctest
    tests = doctest.testmod(verbose=verbose)
    if tests.failed:
        return "☹"*tests.failed + " / " + numtostring(tests.attempted)
    return "^_^ (" + numtostring(tests.attempted) + ")"


if __name__ == "__main__":
    args = parse_args()

    if args.verbosity:
        fcp.node.defaultVerbosity = int(args.verbosity)
    
    if args.host:
        fcp.node.defaultFCPHost = args.host

    if args.user and not args.port:
        dirs = appdirs.AppDirs("babcom", "freenet")
        knownusersfile = os.path.join(dirs.user_data_dir, "knownusers")
        if os.path.isfile(knownusersfile):
            users = []
            with open(knownusersfile) as f:
                for line in f:
                    l = line.split()
                    if l:
                        identity, port = l[:2]
                        userstring = " ".join(l[2:]).strip() + "".join(["@"] + l[:1])
                        if userstring.startswith(args.user):
                            users.append((userstring, int(port)))
            if len(users) == 1:
                userstring = users[0][0]
                args.port = users[0][1]
            if users[1:]:
                choice = None
                print("more than one identity with name", args.user, "please select one.")
                while choice is None:
                    for i in range(len(users)):
                        print(i+1, users[i][0])
                    res = input("Insert the number of the identity to use (1 to " + str(len(users)) + "): ")
                    try:
                        choice = int(res)
                    except ValueError:
                        print("not a number")
                    if choice < 1 or len(users) < choice:
                        choice = None
                        print("the number is not in the range", str(i+1), "to", str(len(userss)))
                userstring = users[choice][0]
                args.port = users[choice][1]
            if users:
                logging.info("Using port %s for user %s", args.port, userstring)
        
    if args.no_spawn:
        args.spawn = False
        if args.port:
            fcp_port = args.port
        else:
            fcp_port = fcp.node.defaultFCPPort
    else: # TODO: if no port given, see whether we know a port for args.user
        port = freenet.spawn.choose_free_port(
            fcp.node.defaultFCPHost,
            (int(args.port) if args.port else 9481))
        logging.info("Trying to spawn a node on port {}.".format(port))
        fcp_port = freenet.spawn.spawn_node(
            fcp_port=port, transient=args.transient)
        logging.info("Spawned node on port {}.".format(fcp_port))
    fcp.node.defaultFCPPort = fcp_port
    
    slowtests = args.slowtests
    if args.test:
        in_doctest = True
        if args.verbosity:
            print(_test(verbose=True))
        else:
            print(_test())
        in_doctest = False
        sys.exit(0)

    prompt = Babcom()
    prompt.username = args.user
    prompt.recover = args.recover
    prompt.transient = args.transient
    prompt.spawn = not args.no_spawn
    prompt.fcp_port = fcp_port
    
    try:
        prompt.cmdloop('Starting babcom, type help or intro')
    finally: # ensure that spawns get stopped at exit, even if something went wrong
        if prompt.spawn:
            try:
                with fcp.node.FCPNode(port=prompt.fcp_port) as n:
                    if n.nodeIsAlive:
                        logging.warning("Babcom was killed. Cleanly shutting down node.")
                        freenet.spawn.teardown_node(prompt.fcp_port,
                                                    delete_node_folder=prompt.transient)
            except Exception:   # likely already closed cleanly: Any
                                # exception here signifies successful
                                # teardown in the comdloop.
                pass