keyring.c

/*
Copyright (C) 2016-2018 Flinders University
Copyright (C) 2013-2015 Serval Project Inc.
Copyright (C) 2010-2012 Paul Gardner-Stephen
 
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
 
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
 
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#include <stdio.h>
#include <assert.h>
#include "serval.h"
#include "conf.h"
#include "constants.h"
#include "overlay_buffer.h"
#include "overlay_address.h"
#include "crypto.h"
#include "keyring.h"
#include "dataformats.h"
#include "str.h"
#include "mem.h"
#include "rotbuf.h"
#include "route_link.h"
#include "commandline.h"
#include "debug.h"

static keyring_file *keyring_open_or_create(const char *path, int writeable);
static int keyring_initialise(keyring_file *k);
static int keyring_load(keyring_file *k, const char *pin);
static keyring_file *keyring_open_create_instance(const char *pin, int force_create);
static void keyring_free_keypair(keypair *kp);
static int keyring_identity_mac(const keyring_identity *id, unsigned char *pkrsalt, unsigned char *mac);
static int keyring_commit_identity(keyring_file *k, keyring_identity *id);

struct combined_pk{
  identity_t sign_key;
  sid_t box_key;
};

struct combined_sk{
  sign_keypair_t sign_key;
  uint8_t box_key[crypto_box_SECRETKEYBYTES];
};

static int _keyring_open(keyring_file *k, const char *path, const char *mode)
{
  DEBUGF(keyring, "opening %s in \"%s\" mode", alloca_str_toprint(path), mode);
  if (sodium_init()==-1)
    return WHY("Failed to initialise libsodium");
  k->file = fopen(path, mode);
  if (!k->file) {
    if (errno != EPERM && errno != ENOENT)
      return WHYF_perror("fopen(%s, \"%s\")", alloca_str_toprint(path), mode);
    DEBUGF(keyring, "cannot open %s in \"%s\" mode", alloca_str_toprint(path), mode);
  }
  return 0;
}

/*
 * Open keyring file and  detect its size.
 */
static keyring_file *keyring_open_or_create(const char *path, int writeable)
{
  /* Allocate structure */
  keyring_file *k = emalloc_zero(sizeof(keyring_file));
  if (!k)
    return NULL;
  /* Open keyring file read-write if we can, else use it read-only, else create it. */
  if (writeable && _keyring_open(k, path, "r+") == -1) {
    keyring_free(k);
    return NULL;
  }
  if (!k->file && _keyring_open(k, path, "r") == -1) {
    keyring_free(k);
    return NULL;
  }
  if (!k->file && writeable && _keyring_open(k, path, "w+") == -1) {
    keyring_free(k);
    return NULL;
  }
  if (!k->file) {
    WHYF_perror("cannot open or create keyring file %s", alloca_str_toprint(path));
    keyring_free(k);
    return NULL;
  }
  if (fseeko(k->file, 0, SEEK_END)) {
    WHYF_perror("fseeko(%s, 0, SEEK_END)", alloca_str_toprint(path));
    keyring_free(k);
    return NULL;
  }
  k->file_size = ftello(k->file);
  return k;
}

/*
 * Write initial content of keyring file (erasing anything already there).
 */
static int keyring_initialise(keyring_file *k)
{
  // Write 2KB of zeroes, followed by 2KB of random bytes as salt.
  if (fseeko(k->file, 0, SEEK_SET))
    return WHYF_perror("fseeko(%d, 0, SEEK_SET)", fileno(k->file));
  unsigned char buffer[KEYRING_PAGE_SIZE];
  bzero(&buffer[0], KEYRING_BAM_BYTES);
  randombytes_buf(&buffer[KEYRING_BAM_BYTES], KEYRING_PAGE_SIZE - KEYRING_BAM_BYTES);
  if (fwrite(buffer, KEYRING_PAGE_SIZE, 1, k->file) != 1) {
    WHYF_perror("fwrite(%p, %zu, 1, %d)", buffer, KEYRING_PAGE_SIZE - KEYRING_BAM_BYTES, fileno(k->file));
    return WHYF("Could not write page into keyring file");
  }
  k->file_size = KEYRING_PAGE_SIZE;
  return 0;
}

/*
 * Read the BAM and create initial context using the stored salt.
 */
static int keyring_load(keyring_file *k, const char *pin)
{
  assert(k->bam == NULL);
  /* Read BAMs for each slab in the file */
  keyring_bam **b=&k->bam;
  size_t offset = 0;
  while (offset < k->file_size) {
    /* Read allocmap from slab.  If offset is zero, read the salt */
    if (fseeko(k->file, (off_t)offset, SEEK_SET)) {
      WHYF_perror("fseeko(%d, %zd, SEEK_SET)", fileno(k->file), offset);
      return WHY("Could not seek to BAM in keyring file");
    }
    *b = emalloc_zero(sizeof(keyring_bam));
    if (!*b)
      return WHYF("Could not allocate keyring_bam structure");
    (*b)->file_offset = offset;
    /* Read allocation bitmap */
    int r = fread((*b)->allocmap, KEYRING_BAM_BYTES, 1, k->file);
    if (r != 1) {
      WHYF_perror("fread(%p, %zd, 1, %d)", (*b)->allocmap, KEYRING_BAM_BYTES, fileno(k->file));
      return WHYF("Could not read BAM from keyring file");
    }
    /* Read salt if this is the first allocmap block.
       We setup a context for this self-supplied key-ring salt.
       (other keyring salts may be provided later on, resulting in
       multiple contexts being loaded) */
    if (!offset) {
      k->KeyRingPin = str_edup(pin);
      k->KeyRingSaltLen=KEYRING_PAGE_SIZE-KEYRING_BAM_BYTES;
      k->KeyRingSalt = emalloc(k->KeyRingSaltLen);
      if (!k->KeyRingSalt)
	return WHYF("Could not allocate keyring_context->salt");
      r = fread(k->KeyRingSalt, k->KeyRingSaltLen, 1, k->file);
      if (r!=1) {
	WHYF_perror("fread(%p, %d, 1, %d)", k->KeyRingSalt, k->KeyRingSaltLen, fileno(k->file));
	return WHYF("Could not read salt from keyring file");
      }
    }
    /* Skip to next slab, and find next bam pointer. */
    offset += KEYRING_PAGE_SIZE * (KEYRING_BAM_BYTES << 3);
    b = &(*b)->next;
  }
  return 0;
}

static unsigned is_slot_allocated(const keyring_file *k, unsigned slot)
{
  assert(slot != 0);
  assert(slot < KEYRING_BAM_BITS);
  unsigned position = slot & (KEYRING_BAM_BITS - 1);
  unsigned byte = position >> 3;
  uint8_t mask = 1 << (position & 7);
  return (k->bam->allocmap[byte] & mask) ? 1 : 0;
}

static unsigned is_slot_loadable(const keyring_file *k, unsigned slot)
{
  assert(slot != 0);
  assert(slot < KEYRING_BAM_BITS);
  unsigned position = slot & (KEYRING_BAM_BITS - 1);
  unsigned byte = position >> 3;
  uint8_t mask = 1 << (position & 7);
  return (k->bam->allocmap[byte] & mask) && !(k->bam->loadmap[byte] & mask);
}

static void mark_slot_allocated(keyring_file *k, unsigned slot, int allocated)
{
  assert(slot != 0);
  assert(slot < KEYRING_BAM_BITS);
  unsigned position = slot & (KEYRING_BAM_BITS - 1);
  unsigned byte = position >> 3;
  uint8_t mask = 1 << (position & 7);
  if (allocated) {
    k->bam->allocmap[byte] |= mask;
  } else {
    assert(k->bam->allocmap[byte] & mask); // already marked as allocated
    k->bam->allocmap[byte] &= ~mask;
  }
}

static void mark_slot_loaded(keyring_file *k, unsigned slot, int loaded)
{
  assert(slot != 0);
  assert(slot < KEYRING_BAM_BITS);
  unsigned position = slot & (KEYRING_BAM_BITS - 1);
  unsigned byte = position >> 3;
  uint8_t mask = 1 << (position & 7);
  if (loaded) {
    k->bam->loadmap[byte] |= mask;
  } else {
    assert(k->bam->loadmap[byte] & mask); // already marked as loaded
    k->bam->loadmap[byte] &= ~mask;
  }
}

void keyring_iterator_start(keyring_file *k, keyring_iterator *it)
{
  assert(k);
  bzero(it, sizeof(keyring_iterator));
  it->file = k;
}

keyring_identity * keyring_next_identity(keyring_iterator *it)
{
  assert(it->file);
  if (!it->identity)
    it->identity=it->file->identities;
  else
    it->identity=it->identity->next;
  if (it->identity)
    it->keypair = it->identity->keypairs;
  else
    it->keypair = NULL;
  return it->identity;
}

keypair * keyring_next_key(keyring_iterator *it)
{
  if (it->keypair)
    it->keypair = it->keypair->next;
  if (!it->keypair)
    keyring_next_identity(it);
  return it->keypair;
}

keypair *keyring_next_keytype(keyring_iterator *it, enum keyring_keytype keytype)
{
  keypair *kp;
  while((kp=keyring_next_key(it)) && kp->type!=keytype)
    ;
  return kp;
}

keypair *keyring_identity_keytype(const keyring_identity *id, enum keyring_keytype keytype)
{
  keypair *kp=id->keypairs;
  while(kp && kp->type!=keytype)
    kp=kp->next;
  return kp;
}

keypair *keyring_find_did(keyring_iterator *it, const char *did)
{
  keypair *kp;
  while((kp=keyring_next_keytype(it, KEYTYPE_DID))){
    if ((!did[0])
	||(did[0]=='*'&&did[1]==0)
	||(!strcasecmp(did,(char *)kp->private_key))
    ) {
      return kp;
    }
  }
  return NULL;
}

keyring_identity *keyring_find_identity_sid(keyring_file *k, const sid_t *sidp){
  keyring_identity *id = k->identities;
  while(id && (!id->box_pk || cmp_sid_t(id->box_pk,sidp)!=0))
    id = id->next;
  return id;
}

keyring_identity *keyring_find_identity(keyring_file *k, const identity_t *sign){
  keyring_identity *id = k->identities;
  while(id && (!id->box_pk || cmp_identity_t(&id->sign_keypair->public_key, sign)!=0))
    id = id->next;
  return id;
}

static void add_subscriber(keyring_identity *id)
{
  id->subscriber = find_subscriber(id->box_pk->binary, SID_SIZE, 1);
  if (id->subscriber) {
    // TODO flag for unroutable identities...?
    if (id->subscriber->reachable == REACHABLE_NONE)
      id->subscriber->reachable = REACHABLE_SELF;
    id->subscriber->identity = id;

    // copy our signing key, so we can pass it to peers
    id->subscriber->id_public = id->sign_keypair->public_key;
    id->subscriber->id_valid = 1;

    keypair *kp = id->keypairs;
    while(kp){
      if (kp->type == KEYTYPE_CRYPTOCOMBINED){
	id->subscriber->id_combined = 1;
	break;
      }
      kp = kp->next;
    }
  }
}

static void wipestr(char *str)
{
  while (*str)
    *str++ = ' ';
}

/* Wipe any sensitive data from the given identity and release its allocated memory by calling
 * free(3).  The identity must not be linked into any keyring, but this function does not check
 * that, so it is only for internal use within keyring.c.
 */
static void free_identity(keyring_identity *id)
{
  if (id->PKRPin) {
    wipestr(id->PKRPin);
    free(id->PKRPin);
    id->PKRPin = NULL;
  }
  while(id->keypairs){
    keypair *kp=id->keypairs;
    id->keypairs=kp->next;
    keyring_free_keypair(kp);
  }
  if (id->challenge)
    free(id->challenge);
  if (id->subscriber)
    link_stop_routing(id->subscriber);
  bzero(id,sizeof(keyring_identity));
  free(id);
}

void keyring_free(keyring_file *k)
{
  if (!k) return;

  /* Close keyring file handle */
  if (k->file) {
    fclose(k->file);
    k->file = NULL;
  }

  /* Free BAMs (no substructure, so easy) */
  while (k->bam) {
    keyring_bam *b = k->bam;
    k->bam = b->next;
    bzero(b, sizeof(keyring_bam));
    free(b);
  }

  /* Free dynamically allocated salt strings.
     Don't forget to overwrite any private data. */
  if (k->KeyRingPin) {
    /* Wipe pin from local memory before freeing. */
    wipestr(k->KeyRingPin);
    free(k->KeyRingPin);
    k->KeyRingPin = NULL;
  }
  if (k->KeyRingSalt) {
    bzero(k->KeyRingSalt,k->KeyRingSaltLen);
    free(k->KeyRingSalt);
    k->KeyRingSalt = NULL;
    k->KeyRingSaltLen = 0;
  }
  
  /* Wipe out any loaded identities */
  while(k->identities){
    keyring_identity *i = k->identities;
    k->identities=i->next;
    free_identity(i);
  }
  
  /* Wipe everything, just to be sure. */
  bzero(k,sizeof(keyring_file));
  free(k);
  
  return;
}

/* Release ("lock") all the identities that have a given PIN, by unlinking them from the in-memory
 * cache list.  DOES call free_identity(id) on each released identity.  When this function returns,
 * there are no unlocked identities with the given PIN.
 */
void keyring_release_identities_by_pin(keyring_file *k, const char *pin)
{
  INFO("release identity by PIN");
  DEBUGF(keyring, "release identity PIN=%s", alloca_str_toprint(pin));
  keyring_identity **i=&k->identities;
  while(*i){
    keyring_identity *id = *i;
    if (id->PKRPin && strcmp(id->PKRPin, pin) == 0) {
      INFOF("release identity slot=%u SID=%s", id->slot, alloca_tohex_sid_t(*id->box_pk));
      *i = id->next;
      mark_slot_loaded(k, id->slot, 0);
      free_identity(id);
    }else{
      i=&id->next;
    }
  }
}

/* Release the given single identity by unlinking it from the in-memory cache list.  To ensure that
 * re-entering the identity's PIN will unlock it again, mark any other unlocked identities that have
 * the same PIN as no longer "fully" unlocked, so that keyring_enter_pin() will re-try the
 * decryption.  Does NOT call free_identity(id), so the identity's in-memory structure remain
 * intact; the caller is responsible for freeing the identity.
 */
void keyring_release_identity(keyring_file *k, keyring_identity *id)
{
  INFOF("release identity slot=%u SID=%s", id->slot, alloca_tohex_sid_t(*id->box_pk));
  keyring_identity **prev = NULL;
  keyring_identity **i;
  // find the identity in the keyring's linked list, so it can be unlinked
  for (i = &k->identities; *i; i = &(*i)->next) {
    keyring_identity *iid = *i;
    if (iid == id)
      prev = i;
    // mark any other identities that have the same PIN as no longer fully unlocked
    if (id->PKRPin && iid->PKRPin && strcmp(id->PKRPin, iid->PKRPin) == 0)
      iid->is_fully_unlocked = 0;
  }
  assert(prev); // the identity being released must be in the keyring
  (*prev) = id->next;
  id->next = NULL;
  mark_slot_loaded(k, id->slot, 0);
}

/* Release the single identity with the given SID, by unlinking it from the in-memory cache list.
 * See the comment on keyring_release_identity() regarding PIN management.  Returns zero if an
 * identity was found and released, -1 if no such identity was found.  Unlike
 * keyring_release_identity(), this function DOES call free_identity(id) on any released identity
 * before returning.
 */
int keyring_release_subscriber(keyring_file *k, const sid_t *sid)
{
  INFOF("release identity SID=%s", alloca_tohex_sid_t(*sid));
  keyring_identity **i;
  for (i = &k->identities; *i; i = &(*i)->next) {
    keyring_identity *iid = *i;
    if (cmp_sid_t(iid->box_pk, sid) == 0) {
      keyring_release_identity(k, iid);
      free_identity(iid);
      return 0;
    }
  }
  return WHYF("cannot release non-existent keyring entry SID=%s", alloca_tohex_sid_t(*sid));
}

/* Free the given identity after ensuring that it is not linked into the given keyring.
 */
void keyring_free_identity(keyring_file *k, keyring_identity *id)
{
  keyring_identity *i;
  for (i = k->identities; i; i = i->next)
    assert(i != id);
  assert(id->next == NULL);
  free_identity(id);
}

/*
  En/Decrypting a block requires use of the first 32 bytes of the block to provide
  salt.  The next 64 bytes constitute a message authentication code (MAC) that is
  used to verify the validity of the block.  The verification occurs in a higher
  level function, and all we need to know here is that we shouldn't decrypt the
  first 96 bytes of the block.
*/
static int keyring_munge_block(
  unsigned char *block, int len /* includes the first 96 bytes */,
  unsigned char *KeyRingSalt, int KeyRingSaltLen,
  const char *KeyRingPin, const char *PKRPin)
{
  DEBUGF(keyring, "KeyRingPin=%s PKRPin=%s", alloca_str_toprint(KeyRingPin), alloca_str_toprint(PKRPin));
  int exit_code=1;
  unsigned char hashKey[crypto_hash_sha512_BYTES];
  unsigned char hashNonce[crypto_hash_sha512_BYTES];

  unsigned char work[65536];

  if (len<96) return WHY("block too short");

  unsigned char *PKRSalt=&block[0];
  int PKRSaltLen=32;

#if crypto_box_SECRETKEYBYTES>crypto_hash_sha512_BYTES
#error crypto primitive key size too long -- hash needs to be expanded
#endif
#if crypto_box_NONCEBYTES>crypto_hash_sha512_BYTES
#error crypto primitive nonce size too long -- hash needs to be expanded
#endif

  /* Generate key and nonce hashes from the various inputs */
  unsigned ofs;
#define APPEND(buf, len) { \
    assert(ofs <= sizeof work); \
    unsigned __len = (len); \
    if (__len > sizeof work - ofs) { \
      WHY("Input too long"); \
      goto kmb_safeexit; \
    } \
    bcopy((buf), &work[ofs], __len); \
    ofs += __len; \
  }
  /* Form key as hash of various concatenated inputs.
     The ordering and repetition of the inputs is designed to make rainbow tables
     infeasible */
  ofs=0;
  APPEND(PKRSalt,PKRSaltLen);
  if (PKRPin)
    APPEND(PKRPin,strlen(PKRPin));
  APPEND(PKRSalt,PKRSaltLen);
  APPEND(KeyRingPin,strlen(KeyRingPin));
  crypto_hash_sha512(hashKey,work,ofs);

  /* Form the nonce as hash of various other concatenated inputs */
  ofs=0;
  APPEND(KeyRingPin,strlen(KeyRingPin));
  APPEND(KeyRingSalt,KeyRingSaltLen);
  APPEND(KeyRingPin,strlen(KeyRingPin));
  if (PKRPin)
    APPEND(PKRPin,strlen(PKRPin));
  crypto_hash_sha512(hashNonce,work,ofs);

  /* Now en/de-crypt the remainder of the block.
     We do this in-place for convenience, so you should not pass in a mmap()'d
     lump. */
  crypto_stream_xsalsa20_xor(&block[96],&block[96],len-96, hashNonce,hashKey);
  exit_code=0;

 kmb_safeexit:
  /* Wipe out all sensitive structures before returning */
  ofs=0;
  bzero(&work[0],65536);
  bzero(&hashKey[0],crypto_hash_sha512_BYTES);
  bzero(&hashNonce[0],crypto_hash_sha512_BYTES);
  return exit_code;
#undef APPEND
}

const char *keytype_str(enum keyring_keytype ktype, const char *unknown)
{
  switch (ktype) {
  case KEYTYPE_INVALID: return "INVALID";
  case KEYTYPE_CRYPTOBOX: return "CRYPTOBOX";
  case KEYTYPE_CRYPTOSIGN: return "CRYPTOSIGN";
  case KEYTYPE_RHIZOME: return "RHIZOME";
  case KEYTYPE_DID: return "DID";
  case KEYTYPE_PUBLIC_TAG: return "PUBLIC_TAG";
  case KEYTYPE_CRYPTOCOMBINED: return "CRYPTOCOMBINED";
  default: return unknown;
  }
}

struct key_type {
  size_t public_key_size;
  size_t private_key_size;
  size_t packed_size;
  void (*creator)(keypair *);
  int (*packer)(const keypair *, struct rotbuf *);
  int (*unpacker)(keypair *, struct rotbuf *, size_t);
  void (*dumper)(const keypair *, XPRINTF, int);
  int (*loader)(keypair *, const char *);
};

static void create_rhizome(keypair *kp)
{
  randombytes_buf(kp->private_key, kp->private_key_len);
}

static void create_cryptocombined(keypair *kp)
{
  struct combined_pk *pk = (struct combined_pk *)kp->public_key;
  struct combined_sk *sk = (struct combined_sk *)kp->private_key;
  crypto_sign_ed25519_keypair(pk->sign_key.binary, sk->sign_key.binary);
  crypto_sign_ed25519_sk_to_curve25519(sk->box_key, sk->sign_key.binary);
  crypto_scalarmult_base(pk->box_key.binary, sk->box_key);
}

static int pack_cryptocombined(const keypair *kp, struct rotbuf *rb)
{
  sign_private_t seed;
  struct combined_sk *sk = (struct combined_sk *)kp->private_key;
  crypto_sign_ed25519_sk_to_seed(seed.binary, sk->sign_key.binary);
  rotbuf_putbuf(rb, seed.binary, sizeof seed);
  return 0;
}

static int unpack_cryptocombined(keypair *kp, struct rotbuf *rb, size_t key_length)
{
  sign_private_t seed;
  if (key_length != sizeof seed)
    return -1;
  struct combined_pk *pk = (struct combined_pk *)kp->public_key;
  struct combined_sk *sk = (struct combined_sk *)kp->private_key;
  rotbuf_getbuf(rb, seed.binary, sizeof seed);
  crypto_sign_ed25519_seed_keypair(pk->sign_key.binary, sk->sign_key.binary, seed.binary);
  crypto_sign_ed25519_sk_to_curve25519(sk->box_key, sk->sign_key.binary);
  crypto_scalarmult_base(pk->box_key.binary, sk->box_key);
  return 0;
}

static int pack_private_only(const keypair *kp, struct rotbuf *rb)
{
  rotbuf_putbuf(rb, kp->private_key, kp->private_key_len);
  return 0;
}

static int pack_public_only(const keypair *kp, struct rotbuf *rb)
{
  rotbuf_putbuf(rb, kp->public_key, kp->public_key_len);
  return 0;
}

static int pack_private_public(const keypair *kp, struct rotbuf *rb)
{
  rotbuf_putbuf(rb, kp->private_key, kp->private_key_len);
  rotbuf_putbuf(rb, kp->public_key, kp->public_key_len);
  return 0;
}

static void dump_private_public(const keypair *kp, XPRINTF xpf, int include_secret)
{
  if (kp->public_key_len)
    xprintf(xpf, " pub=%s", alloca_tohex(kp->public_key, kp->public_key_len));
  if (include_secret && kp->private_key_len)
    xprintf(xpf, " sec=%s", alloca_tohex(kp->private_key, kp->private_key_len));
}

static int _load_decode_hex(const char **hex, unsigned char **buf, size_t *len)
{
  const char *end = NULL;
  size_t hexlen = strn_fromhex(NULL, -1, *hex, &end);
  if (hexlen == 0 || end == NULL || (*end != ' ' && *end != '\0'))
    return WHY("malformed hex value");
  if (*len == 0) {
    assert(*buf == NULL);
    *len = hexlen;
    if ((*buf = emalloc_zero(*len)) == NULL)
      return -1;
  }
  else if (hexlen != *len)
    return WHYF("invalid hex value, incorrect length (expecting %zu bytes, got %zu)", *len, hexlen);
  strn_fromhex(*buf, *len, *hex, hex);
  assert(*hex == end);
  return 0;
}

static int load_private_public(keypair *kp, const char *text)
{
  assert(kp->public_key_len != 0);
  assert(kp->public_key != NULL);
  assert(kp->private_key_len != 0);
  assert(kp->private_key != NULL);
  const char *t = text;
  int got_pub = 0;
  int got_sec = 0;
  while (*t) {
    while (isspace(*t))
      ++t;
    if (str_startswith(t, "pub=", &t)) {
      if (_load_decode_hex(&t, &kp->public_key, &kp->public_key_len) == -1)
	WHY("cannot decode pub= field");
      else
	got_pub = 1;
    }
    else if (str_startswith(t, "sec=", &t)) {
      if (_load_decode_hex(&t, &kp->private_key, &kp->private_key_len) == -1)
	WHY("cannot decode sec= field");
      else
	got_sec = 1;
    }
    else if (*t)
      return WHYF("unsupported dump field: %s", t);
  }
  if (!got_sec)
    return WHY("missing sec= field");
  if (!got_pub)
    return WHY("missing pub= field");
  return 0;
}

static int load_private(keypair *kp, const char *text)
{
  assert(kp->private_key_len != 0);
  assert(kp->private_key != NULL);
  const char *t = text;
  int got_sec = 0;
  while (*t) {
    while (isspace(*t))
      ++t;
    if (str_startswith(t, "sec=", &t)) {
      if (_load_decode_hex(&t, &kp->private_key, &kp->private_key_len) == -1)
	WHY("cannot decode sec= field");
      else
	got_sec = 1;
    } else if (str_startswith(t, "pub=", &t)) {
      WARN("skipping pub= field");
      while (*t && !isspace(*t))
	++t;
    }
    else if (*t)
      return WHYF("unsupported dump field: %s", t);
  }
  if (!got_sec)
    return WHY("missing sec= field");
  return 0;
}

static int load_cryptobox(keypair *kp, const char *text)
{
  if (load_private(kp, text) == -1)
    return -1;
  crypto_scalarmult_base(kp->public_key, kp->private_key);
  return 0;
}

static int load_private_only(keypair *kp, const char *text)
{
  assert(kp->public_key_len == 0);
  assert(kp->public_key == NULL);
  return load_private(kp, text);
}

static int load_unknown(keypair *kp, const char *text)
{
  assert(kp->private_key_len == 0);
  assert(kp->private_key == NULL);
  assert(kp->public_key_len == 0);
  assert(kp->public_key == NULL);
  const char *t = text;
  while (*t) {
    while (isspace(*t))
      ++t;
    if (str_startswith(t, "pub=", &t)) {
      if (_load_decode_hex(&t, &kp->public_key, &kp->public_key_len) == -1)
	WHY("cannot decode pub= field");
    }
    else if (str_startswith(t, "sec=", &t)) {
      if (_load_decode_hex(&t, &kp->private_key, &kp->private_key_len) == -1)
	WHY("cannot decode sec= field");
    }
    else if (*t)
      return WHYF("unsupported dump field: %s", t);
  }
  return 0;
}

static int unpack_private_public(keypair *kp, struct rotbuf *rb, size_t key_length)
{
  if(key_length != kp->private_key_len + kp->public_key_len)
    return -1;
  rotbuf_getbuf(rb, kp->private_key, kp->private_key_len);
  rotbuf_getbuf(rb, kp->public_key, kp->public_key_len);
  return 0;
}

static int unpack_private_only(keypair *kp, struct rotbuf *rb, size_t key_length)
{
  if (!kp->private_key){
    kp->private_key_len = key_length;
    if ((kp->private_key = emalloc(kp->private_key_len))==NULL)
      return -1;
  }else{
    if (kp->private_key_len != key_length)
      return -1;
  }
  rotbuf_getbuf(rb, kp->private_key, kp->private_key_len);
  return 0;
}

static int unpack_public_only(keypair *kp, struct rotbuf *rb, size_t key_length)
{
  if (!kp->public_key){
    kp->public_key_len = key_length;
    if ((kp->public_key = emalloc(kp->public_key_len))==NULL)
      return -1;
  }else{
    if(kp->public_key_len != key_length)
      return -1;
  }
  rotbuf_getbuf(rb, kp->public_key, kp->public_key_len);
  return 0;
}

static int unpack_cryptobox(keypair *kp, struct rotbuf *rb, size_t key_length)
{
  if (key_length != kp->private_key_len)
    return -1;
  rotbuf_getbuf(rb, kp->private_key, kp->private_key_len);
  if (!rb->wrap)
    crypto_scalarmult_base(kp->public_key, kp->private_key);
  return 0;
}

static int pack_did_name(const keypair *kp, struct rotbuf *rb)
{
  // Ensure DID is nul terminated.
  if (strnchr((const char *)kp->private_key, kp->private_key_len, '\0') == NULL)
    return WHY("DID missing nul terminator");
  // Ensure name is nul terminated.
  if (strnchr((const char *)kp->public_key, kp->public_key_len, '\0') == NULL)
    return WHY("Name missing nul terminator");
  return pack_private_public(kp, rb);
}

static int unpack_did_name(keypair *kp, struct rotbuf *rb, size_t key_length)
{
  if (unpack_private_public(kp, rb, key_length) == -1)
    return -1;
  // Fail if DID and Name are not nul terminated.
  return    strnchr((const char *)kp->private_key, kp->private_key_len, '\0') != NULL
	 && strnchr((const char *)kp->public_key, kp->public_key_len, '\0') != NULL
	 ? 0 : -1;
}

static void dump_did_name(const keypair *kp, XPRINTF xpf, int UNUSED(include_secret))
{
  xprintf(xpf, " DID=%s", alloca_str_toprint_quoted((const char *)kp->private_key, "\"\""));
  xprintf(xpf, " Name=%s", alloca_str_toprint_quoted((const char *)kp->public_key, "\"\""));
}

static int load_did_name(keypair *kp, const char *text)
{
  assert(kp->public_key != NULL);
  assert(kp->private_key != NULL);
  const char *t = text;
  int got_did = 0;
  int got_name = 0;
  while (*t) {
    while (isspace(*t))
      ++t;
    if (str_startswith(t, "DID=\"", &t)) {
      if (got_did)
	return WHY("duplicate DID");
      const char *e = NULL;
      bzero(kp->private_key, kp->private_key_len);
      strn_fromprint((char *)kp->private_key, kp->private_key_len, t, 0, '"', &e);
      if (*e != '"')
	return WHY("malformed DID quoted string");
      t = e + 1;
      got_did = 1;
    } else if (str_startswith(t, "Name=\"", &t)) {
      if (got_name)
	return WHY("duplicate Name");
      const char *e = NULL;
      bzero(kp->public_key, kp->public_key_len);
      strn_fromprint((char *)kp->public_key, kp->public_key_len, t, 0, '"', &e);
      if (*e != '"')
	return WHY("malformed Name quoted string");
      t = e + 1;
      got_name = 1;
    }
    else if (*t)
      return WHYF("unsupported dump content: %s", t);
  }
  if (!got_did)
    return WHY("missing DID");
  if (!got_name)
    return WHY("missing Name");
  return 0;
}

/* This is where all the supported key types are declared.  In order to preserve backward
 * compatibility (reading keyring files from older versions of Serval DNA), DO NOT ERASE OR RE-USE
 * ANY KEY TYPE ENTRIES FROM THIS ARRAY.  If a key type is no longer used, it must be permanently
 * deprecated, ie, recognised and simply skipped.  The packer and unpacker functions can be changed
 * to NULL.
 */
const struct key_type key_types[] = {
  [KEYTYPE_CRYPTOBOX] = {
      /* Only the private key is stored, and the public key (SID) is derived from the private key
       * when the keyring is read.
       */
      .private_key_size = crypto_box_SECRETKEYBYTES,
      .public_key_size = crypto_box_PUBLICKEYBYTES,
      .packed_size = crypto_box_SECRETKEYBYTES,
      .creator = NULL, // deprecated
      .packer = pack_private_only,
      .unpacker = unpack_cryptobox,
      .dumper = dump_private_public,
      .loader = load_cryptobox
    },
  [KEYTYPE_CRYPTOSIGN] = {
      /* The NaCl API does not expose any method to derive a cryptosign public key from its private
       * key, although there must be an internal NaCl function to do so.  Subverting the NaCl API to
       * invoke that function risks incompatibility with future releases of NaCl, so instead the
       * public key is stored redundantly in the keyring.
       */
      .private_key_size = crypto_sign_SECRETKEYBYTES,
      .public_key_size = crypto_sign_PUBLICKEYBYTES,
      .packed_size = crypto_sign_SECRETKEYBYTES + crypto_sign_PUBLICKEYBYTES,
      .creator = NULL, // deprecated
      .packer = pack_private_public,
      .unpacker = unpack_private_public,
      .dumper = dump_private_public,
      .loader = load_private_public
    },
  [KEYTYPE_RHIZOME] = {
      /* The Rhizome Secret (a large, unguessable number) is stored in the private key field, and
       * the public key field is not used.
       */
      .private_key_size = 32,
      .public_key_size = 0,
      .packed_size = 32,
      .creator = create_rhizome,
      .packer = pack_private_only,
      .unpacker = unpack_private_only,
      .dumper = dump_private_public,
      .loader = load_private_only
    },
  [KEYTYPE_DID] = {
      /* The DID is stored in nul-terminated unpacked form in the private key field, and the name in
       * nul-terminated ASCII form in the public key field.
       */
      // DO NOT define the following size fields directly using DID_MAXSIZE or ID_NAME_MAXSIZE,
      // which define the Serval DNA API, but not the keyring file format.  This is to avoid the
      // risk that changing the API might (unintentionally) alter the keyring format, leading to
      // non-back-compatible breakage.
      .private_key_size = 32, // should be >= DID_MAXSIZE + 1
      .public_key_size = 64, // should be >= ID_NAME_MAXSIZE + 1
      .packed_size = 32 + 64,
      .creator = NULL, // not included in a newly created identity
      .packer = pack_did_name,
      .unpacker = unpack_did_name,
      .dumper = dump_did_name,
      .loader = load_did_name
    },
  [KEYTYPE_PUBLIC_TAG] = {
      .private_key_size = 0,
      .public_key_size = 0, // size is derived from the stored key length
      .packed_size = 0,
      .creator = NULL, // not included in a newly created identity
      .packer = pack_public_only,
      .unpacker = unpack_public_only,
      .dumper = dump_private_public,
      .loader = load_unknown
    },
  [KEYTYPE_CRYPTOCOMBINED] = {
      .private_key_size = sizeof (struct combined_sk),
      .public_key_size = sizeof (struct combined_pk),
      .packed_size = crypto_sign_SEEDBYTES,
      .creator = create_cryptocombined,
      .packer = pack_cryptocombined,
      .unpacker = unpack_cryptocombined,
      .dumper = dump_private_public,
      .loader = load_private_public
  }
  // ADD MORE KEY TYPES HERE
};

static void keyring_free_keypair(keypair *kp)
{
  if (kp->private_key) {
    bzero(kp->private_key, kp->private_key_len);
    free(kp->private_key);
  }
  if (kp->public_key) {
    bzero(kp->public_key, kp->public_key_len);
    free(kp->public_key);
  }
  bzero(kp, sizeof(keypair));
  free(kp);
}

static keypair *keyring_alloc_keypair(enum keyring_keytype ktype, size_t len)
{
  assert(ktype != KEYTYPE_INVALID);
  keypair *kp = emalloc_zero(sizeof(keypair));
  if (!kp)
    return NULL;
  kp->type = ktype;
  if (ktype < NELS(key_types)) {
    kp->private_key_len = key_types[ktype].private_key_size;
    kp->public_key_len = key_types[ktype].public_key_size;
  } else {
    kp->private_key_len = len;
    kp->public_key_len = 0;
  }
  if (   (kp->private_key_len && (kp->private_key = emalloc(kp->private_key_len)) == NULL)
      || (kp->public_key_len && (kp->public_key = emalloc(kp->public_key_len)) == NULL)
  ) {
    keyring_free_keypair(kp);
    return NULL;
  }
  return kp;
}

static int keyring_pack_identity(const keyring_identity *id, unsigned char packed[KEYRING_PAGE_SIZE])
{
  /* Convert an identity to a KEYRING_PAGE_SIZE bytes long block that consists of 32 bytes of random
   * salt, a 64 byte (512 bit) message authentication code (MAC) and the list of key pairs. */
  randombytes_buf(packed, PKR_SALT_BYTES);
  /* Calculate MAC */
  if (keyring_identity_mac(id, packed /* pkr salt */, packed + PKR_SALT_BYTES /* write mac in after salt */) == -1)
    return -1;
  /* There was a known plain-text opportunity here: byte 96 must be 0x01, and some other bytes are
   * likely deducible, e.g., the location of the trailing 0x00 byte can probably be guessed with
   * confidence.  Payload rotation will frustrate this attack.
   */
  uint16_t rotation = 0;
#ifndef NO_ROTATION
  rotation=randombytes_random();
#endif
  // The two bytes immediately following the MAC describe the rotation offset.
  packed[PKR_SALT_BYTES + PKR_MAC_BYTES] = rotation >> 8;
  packed[PKR_SALT_BYTES + PKR_MAC_BYTES + 1] = rotation & 0xff;
  /* Pack the key pairs into the rest of the slot as a rotated buffer. */
  struct rotbuf rbuf;
  rotbuf_init(&rbuf,
	    packed + PKR_SALT_BYTES + PKR_MAC_BYTES + 2,
	    KEYRING_PAGE_SIZE - (PKR_SALT_BYTES + PKR_MAC_BYTES + 2),
	    rotation);
  keypair *kp=id->keypairs;
  while(kp && !rbuf.wrap){
    enum keyring_keytype ktype = kp->type;
    const char *kts = keytype_str(ktype, "unknown");
    int (*packer)(const keypair *, struct rotbuf *) = NULL;
    size_t keypair_len=0;
    const struct key_type *kt = &key_types[ktype];
    if (ktype == KEYTYPE_INVALID)
      FATALF("ktype = 0x%02x(%s) in keypair kp=%u", ktype, keytype_str(ktype, "unknown"), kp);
    if (ktype < NELS(key_types)) {
      packer = kt->packer;
      keypair_len = kt->packed_size;
      if (keypair_len==0){
	keypair_len = kp->private_key_len + kp->public_key_len;
      }
    } else {
      packer = pack_private_only;
      keypair_len = kp->private_key_len;
    }
    if (packer == NULL) {
      WARNF("no packer function for key type 0x%02x(%s), omitted from keyring file", ktype, kts);
    } else {
      DEBUGF(keyring, "pack key type = 0x%02x(%s)", ktype, kts);
      // First byte is the key type code.
      rotbuf_putc(&rbuf, ktype);
      // The next two bytes are the key pair length, for forward compatibility: so older software can
      // skip over key pairs with an unrecognised type.  The original four first key types do not
      // store the length, for the sake of backward compatibility with legacy keyring files.  Their
      // entry lengths are hard-coded.
      switch (ktype) {
      case KEYTYPE_CRYPTOBOX:
      case KEYTYPE_CRYPTOSIGN:
      case KEYTYPE_RHIZOME:
      case KEYTYPE_DID:
	break;
      default:
	rotbuf_putc(&rbuf, (keypair_len >> 8) & 0xff);
	rotbuf_putc(&rbuf, keypair_len & 0xff);
	break;
      }
      // The remaining bytes is the key pair in whatever format it uses.
      struct rotbuf rbstart = rbuf;
      if (packer(kp, &rbuf) != 0)
	break;
      // Ensure the correct number of bytes were written.
      unsigned packed = rotbuf_delta(&rbstart, &rbuf);
      if (packed != keypair_len) {
	WHYF("key type 0x%02x(%s) packed wrong length (packed %u, expecting %u)", ktype, kts, packed, (int)keypair_len);
	goto scram;
      }
    }
    kp=kp->next;
  }
  // Final byte is a zero key type code.
  rotbuf_putc(&rbuf, 0x00);
  if (rbuf.wrap > 1) {
    WHY("slot overrun");
    goto scram;
  }
  if (kp) {
    WHY("error filling slot");
    goto scram;
  }
  /* Randomfill the remaining part of the slot to frustrate any known-plain-text attack on the
   * keyring.
   */
  {
    unsigned char *buf;
    size_t len;
    while (rotbuf_next_chunk(&rbuf, &buf, &len))
      randombytes_buf(buf, len);
  }
  return 0;
scram:
  /* Randomfill the entire slot to erase any secret keys that may have found their way into it, to
   * avoid leaking sensitive information out through a possibly re-used memory buffer.
   */
  randombytes_buf(packed, KEYRING_PAGE_SIZE);
  return -1;
}

static int cmp_keypair(const keypair *a, const keypair *b)
{
  int c;
  if (a->type < b->type)
    return -1;
  if (a->type > b->type)
    return 1;
  if (a->public_key_len && !b->public_key_len)
    return -1;
  if (!a->public_key_len && b->public_key_len)
    return 1;
  if (a->public_key_len && b->public_key_len){
    assert(a->public_key != NULL);
    assert(b->public_key != NULL);
    size_t len = a->public_key_len;
    if (len > b->public_key_len)
      len = b->public_key_len;
    c = memcmp(a->public_key, b->public_key, len);
    if (c==0 && a->public_key_len!=b->public_key_len)
      c = a->public_key_len - b->public_key_len;
    if (c)
      return c;
  }
  if (a->private_key_len && !b->private_key_len)
    return -1;
  if (!a->private_key_len && b->private_key_len)
    return 1;
  if (a->private_key_len && b->private_key_len) {
    assert(a->private_key != NULL);
    assert(b->private_key != NULL);
    size_t len = a->private_key_len;
    if (len > b->private_key_len)
      len = b->private_key_len;
    c = memcmp(a->private_key, b->private_key, len);
    if (c==0 && a->private_key_len!=b->private_key_len)
      c = a->private_key_len - b->private_key_len;
    if (c)
      return c;
  }
  return 0;
}

/* Ensure that regardless of the order in the keyring file or loaded dump, keypairs are always
 * stored in memory in ascending order of (key type, public key, private key).
 */
static int keyring_identity_add_keypair(keyring_identity *id, keypair *kp)
{
  assert(id);
  assert(kp);
  keypair **ptr=&id->keypairs;
  int c = 1;
  while(*ptr && (c = cmp_keypair(*ptr, kp)) < 0)
    ptr = &(*ptr)->next;
  if (c == 0)
    return 0; // duplicate not inserted
  kp->next = *ptr;
  *ptr = kp;
  return 1;
}

static keyring_identity *keyring_unpack_identity(unsigned char *slot_data, const char *pin)
{
  /* Skip salt and MAC */
  keyring_identity *id = emalloc_zero(sizeof(keyring_identity));
  if (!id)
    return NULL;
  if (pin && *pin)
    id->PKRPin = str_edup(pin);
  // The two bytes immediately following the MAC describe the rotation offset.
  uint16_t rotation = (slot_data[PKR_SALT_BYTES + PKR_MAC_BYTES] << 8) | slot_data[PKR_SALT_BYTES + PKR_MAC_BYTES + 1];
  /* Pack the key pairs into the rest of the slot as a rotated buffer. */
  struct rotbuf rbuf;
  rotbuf_init(&rbuf,
	      slot_data + PKR_SALT_BYTES + PKR_MAC_BYTES + 2,
	      KEYRING_PAGE_SIZE - (PKR_SALT_BYTES + PKR_MAC_BYTES + 2),
	      rotation);
  while (!rbuf.wrap) {
    struct rotbuf rbo = rbuf;
    enum keyring_keytype ktype = rotbuf_getc(&rbuf);
    if (rbuf.wrap || ktype == KEYTYPE_INVALID)
      break; // End of data, stop looking
    size_t keypair_len;
    // No length bytes after the original four key types, for backward compatibility.  All other key
    // types are followed by a two-byte keypair length.
    switch (ktype) {
    case KEYTYPE_CRYPTOBOX:
    case KEYTYPE_CRYPTOSIGN:
    case KEYTYPE_RHIZOME:
    case KEYTYPE_DID:
      keypair_len = key_types[ktype].packed_size;
      break;
    case KEYTYPE_INVALID:
      FATAL("invalid");
    default:
      keypair_len = rotbuf_getc(&rbuf) << 8;
      keypair_len |= rotbuf_getc(&rbuf);
      break;
    }
    if (keypair_len > rotbuf_remain(&rbuf)) {
      DEBUGF(keyring, "invalid keypair length %zu", keypair_len);
      free_identity(id);
      return NULL;
    }
    // Create keyring entry to hold the key pair.  Even entries of unknown type are stored,
    // so they can be dumped.
    keypair *kp = keyring_alloc_keypair(ktype, keypair_len);
    if (kp == NULL) {
      free_identity(id);
      return NULL;
    }
    struct rotbuf rbstart = rbuf;
    int (*unpacker)(keypair *, struct rotbuf *, size_t) = NULL;
    if (ktype < NELS(key_types))
      unpacker = key_types[ktype].unpacker;
    else
      unpacker = unpack_private_only;

    DEBUGF(keyring, "unpack key type = 0x%02x(%s) at offset %u", ktype, keytype_str(ktype, "unknown"), (int)rotbuf_position(&rbo));
    if (unpacker(kp, &rbuf, keypair_len) != 0) {
      // If there is an error, it is probably an empty slot.
      DEBUGF(keyring, "key type 0x%02x does not unpack", ktype);
      keyring_free_keypair(kp);
      free_identity(id);
      return NULL;
    }
    // Ensure that the correct number of bytes was consumed.
    size_t unpacked = rotbuf_delta(&rbstart, &rbuf);
    if (unpacked != keypair_len) {
      // If the number of bytes unpacked does not match the keypair length, it is probably an
      // empty slot.
      DEBUGF(keyring, "key type 0x%02x unpacked wrong length (unpacked %u, expecting %u)", ktype, (int)unpacked, (int)keypair_len);
      keyring_free_keypair(kp);
      free_identity(id);
      return NULL;
    }
    // Got a valid key pair!  Sort the key pairs by (key type, public key, private key) and weed
    // out duplicates.
    if (!keyring_identity_add_keypair(id, kp))
      keyring_free_keypair(kp);
  }
  // If the buffer offset overshot, we got an invalid keypair code and length combination.
  if (rbuf.wrap > 1) {
    DEBUGF(keyring, "slot overrun by %u bytes", rbuf.wrap - 1);
    free_identity(id);
    return NULL;
  }
  DEBUGF(keyring, "unpacked key pairs");
  return id;
}

static int keyring_identity_mac(const keyring_identity *id, unsigned char *pkrsalt, unsigned char *mac)
{
  unsigned char work[65536];
  unsigned ofs = 0;
#define APPEND(buf, len) { \
    assert(ofs <= sizeof work); \
    unsigned __len = (len); \
    if (__len > sizeof work - ofs) { \
      bzero(work, ofs); \
      DEBUG(keyring, "Input too long"); \
      return -1; \
    } \
    bcopy((buf), &work[ofs], __len); \
    ofs += __len; \
  }
  APPEND(&pkrsalt[0], 32);
  keypair *kp=id->keypairs;
  uint8_t found = 0;
  while(kp){
    if (kp->type == KEYTYPE_CRYPTOBOX || kp->type == KEYTYPE_CRYPTOCOMBINED){
      APPEND(kp->private_key, kp->private_key_len);
      APPEND(kp->public_key, kp->public_key_len);
      found = 1;
    }
    kp = kp->next;
  }
  if (!found){
    DEBUG(keyring,"Identity does not have a primary key");
    return -1;
  }
  if (id->PKRPin)
    APPEND(id->PKRPin, strlen(id->PKRPin));
#undef APPEND
  crypto_hash_sha512(mac, work, ofs);
  return 0;
}

static int keyring_finalise_identity(uint8_t *dirty, keyring_identity *id)
{
  keypair *kp = id->keypairs;
  while(kp){
    switch(kp->type){
      case KEYTYPE_INVALID:
	FATALF("ktype = 0x%02x(%s)", kp->type, keytype_str(kp->type, "unknown"));
      case KEYTYPE_CRYPTOBOX:
	id->box_pk = (const sid_t *)kp->public_key;
	id->box_sk = kp->private_key;
	break;
      case KEYTYPE_CRYPTOSIGN:{
	  const sign_keypair_t *keypair = (const sign_keypair_t *)kp->private_key;
	  if (!crypto_isvalid_keypair(&keypair->private_key, &keypair->public_key)){
	    /* SAS key is invalid (perhaps because it was a pre 0.90 format one),
	       so replace it */
	    WARN("SAS key is invalid -- regenerating.");
	    crypto_sign_keypair(kp->public_key, kp->private_key);
	    if (dirty)
	      *dirty = 1;
	  }
	  id->sign_keypair = (const sign_keypair_t *)kp->private_key;
	}
	break;
      case KEYTYPE_CRYPTOCOMBINED:{
	struct combined_pk *pk = (struct combined_pk *)kp->public_key;
	struct combined_sk *sk = (struct combined_sk *)kp->private_key;
	id->box_pk = &pk->box_key;
	id->box_sk = sk->box_key;
	id->sign_keypair = &sk->sign_key;
	break;
      case KEYTYPE_RHIZOME:
      case KEYTYPE_DID:
      case KEYTYPE_PUBLIC_TAG:
	break;
      }
    }
    kp = kp->next;
  }
  return 0;
}


/* Read the slot, and try to decrypt it.  Decryption is symmetric with encryption, so the same
 * function is used for munging the slot before making use of it, whichever way we are going.  Once
 * munged, we then need to verify that the slot is valid, and if so unpack the details of the
 * identity.
 */
static int keyring_decrypt_pkr(keyring_file *k, const char *pin, int slot)
{
  DEBUGF(keyring, "k=%p pin=%s slot=%d", k, alloca_str_toprint(pin), slot);
  unsigned char slot_data[KEYRING_PAGE_SIZE];
  keyring_identity *id=NULL;

  /* 1. Read slot. */
  if (fseeko(k->file,slot*KEYRING_PAGE_SIZE,SEEK_SET))
    return WHY_perror("fseeko");
  if (fread(slot_data, KEYRING_PAGE_SIZE, 1, k->file) != 1)
    return WHY_perror("fread");
  /* 2. Decrypt data from slot. */
  if (keyring_munge_block(slot_data, KEYRING_PAGE_SIZE, k->KeyRingSalt, k->KeyRingSaltLen, k->KeyRingPin, pin)) {
    WHYF("keyring_munge_block() failed, slot=%u", slot);
    goto kdp_safeexit;
  }
  /* 3. Unpack contents of slot into a new identity in the provided context. */
  DEBUGF(keyring, "unpack slot %u", slot);
  if (((id = keyring_unpack_identity(slot_data, pin)) == NULL))
    goto kdp_safeexit; // Not a valid slot
  id->slot = slot;
  /* 4. Verify that slot is self-consistent (check MAC) */
  unsigned char hash[crypto_hash_sha512_BYTES];
  if (keyring_identity_mac(id, slot_data, hash))
    goto kdp_safeexit;
  /* compare hash to record */
  if (memcmp(hash, &slot_data[PKR_SALT_BYTES], crypto_hash_sha512_BYTES)) {
    DEBUGF(keyring, "slot %u is not valid (MAC mismatch)", slot);
    DEBUG_dump(keyring, "computed",hash,crypto_hash_sha512_BYTES);
    DEBUG_dump(keyring, "stored",&slot_data[PKR_SALT_BYTES],crypto_hash_sha512_BYTES);
    goto kdp_safeexit;
  }

  if (!keyring_commit_identity(k, id))
    goto kdp_safeexit;

  INFOF("unlocked identity slot=%u SID=%s", id->slot, alloca_tohex_sid_t(*id->box_pk));
  return 0;

 kdp_safeexit:
  /* Clean up any potentially sensitive data before exiting */
  bzero(slot_data,KEYRING_PAGE_SIZE);
  bzero(hash,crypto_hash_sha512_BYTES);
  if (id)
    free_identity(id);
  return -1;
}

/* Try all valid slots with the PIN and see if we find any identities with that PIN.  We might find
 * none, or more than one.  Returns the total number of unlocked identities with the given PIN,
 * including any that were already unlocked before this function was called.
 */
unsigned keyring_enter_pin(keyring_file *k, const char *pin)
{
  IN();
  DEBUGF(keyring, "k=%p, pin=%s", k, alloca_str_toprint(pin));
  if (!pin) pin="";

  unsigned identity_count = 0;
  bool_t is_fully_unlocked = 1;

  // check if PIN is already entered and all identities with that PIN are currently
  {
    keyring_identity *id;
    for (id = k->identities; id; id = id->next) {
      if (*pin ? (id->PKRPin && strcmp(id->PKRPin, pin) == 0) : (!id->PKRPin)) {
	++identity_count;
	if (!id->is_fully_unlocked)
	  is_fully_unlocked = 0;
      }
    }
  }

  // try to decrypt if there are no identities already open with the given PIN, or if the PIN is not
  // fully unlocked
  if (!identity_count || !is_fully_unlocked) {
    unsigned slot;
    unsigned slots = k->file_size / KEYRING_PAGE_SIZE;
    if (slots >= KEYRING_BAM_BITS)
      slots = KEYRING_BAM_BITS - 1;
    // slot zero is the BAM and salt, so skip it
    for (slot = 1; slot < slots; ++slot) {
       // only try to decrypt slots that are marked as allocated and not already loaded; the cost of
       // decrypting can be up to a second of CPU time on a phone
      if (is_slot_loadable(k, slot)) {
	if (keyring_decrypt_pkr(k, pin, slot) == 0) {
	  mark_slot_loaded(k, slot, 1);
	  ++identity_count;
	}
      }
    }

    // now all identities with the given PIN have been unlocked, so mark the PIN being fully
    // unlocked
    if (!is_fully_unlocked) {
      keyring_identity *id;
      for (id = k->identities; id; id = id->next) {
	if (*pin ? (id->PKRPin && strcmp(id->PKRPin, pin) == 0) : (!id->PKRPin)) {
	  id->is_fully_unlocked = 1;
	}
      }
    }

    if (k->dirty)
      keyring_commit(k);
  }

  RETURN(identity_count);
  OUT();
}

/* Find free slot in keyring.  Slot 0 in any slab is the BAM and possible keyring salt, so only
 * search for space in slots 1 and above.  TODO: Extend to handle more than one slab!
 */
static unsigned find_free_slot(const keyring_file *k)
{
  unsigned i;
  unsigned slot;
  // walk the list of slots, randomising the low order bits of the index
  unsigned mask = randombytes_uniform(KEYRING_ALLOC_CHUNK);
  for (i = 0; i < KEYRING_BAM_BITS; ++i) {
    slot = 1 + (i ^ mask);
    DEBUGF(keyring, "Check %u, is slot %u free?", i, slot);
    if (slot < KEYRING_BAM_BITS && !is_slot_allocated(k, slot))
      return slot;
  }
  return 0;
}

/* Return non-zero if the identity was successfully added, zero if the identity was not added
 * because the SID is already used by an existng identity in the keyring.
 */
static int keyring_commit_identity(keyring_file *k, keyring_identity *id)
{
  assert(id->next == NULL); // not already linked into keyring

  keyring_finalise_identity(&k->dirty, id);
  // Do nothing if an identity with this sid already exists
  if (keyring_find_identity_sid(k, id->box_pk)) {
    DEBUGF(keyring, "identity not committed, SID already in use: SID=%s", alloca_tohex_sid_t(*id->box_pk));
    return 0;
  }
  mark_slot_allocated(k, id->slot, 1);
  mark_slot_loaded(k, id->slot, 1);

  keyring_identity **i=&k->identities;
  while (*i) {
    assert(*i != id); // not already linked into keyring
    i = &(*i)->next;
  }

  *i=id;
  add_subscriber(id);
  DEBUGF(keyring, "identity committed slot=%u SID=%s", id->slot, alloca_tohex_sid_t(id->subscriber->sid));
  return 1;
}

static keyring_identity *keyring_new_identity()
{
  keyring_identity *id = emalloc_zero(sizeof(keyring_identity));
  if (!id)
    return NULL;

  /* Allocate key pairs */
  unsigned ktype;
  for (ktype = 1; ktype < NELS(key_types); ++ktype) {
    if (key_types[ktype].creator) {
      keypair *kp = keyring_alloc_keypair(ktype, 0);
      if (kp == NULL){
	free_identity(id);
	return NULL;
      }
      key_types[ktype].creator(kp);
      keyring_identity_add_keypair(id, kp);
    }
  }
  assert(id->keypairs);
  return id;
}

keyring_identity *keyring_inmemory_identity(){
  keyring_identity *id = keyring_new_identity();
  if (id) {
    keyring_finalise_identity(NULL, id);
    add_subscriber(id);
    INFOF("created in-memory identity SID=%s", alloca_tohex_sid_t(id->subscriber->sid));
  }
  return id;
}

/* Create a new identity in the specified context (which supplies the keyring pin) with the
 * specified PKR pin.  The crypto_box and crypto_sign key pairs are automatically created, and the
 * PKR is packed and written to a hithero unallocated slot which is then marked full.  Requires an
 * explicit call to keyring_commit()
*/
keyring_identity *keyring_create_identity(keyring_file *k, const char *pin)
{
  DEBUGF(keyring, "k=%p", k);
  if (!k->bam) {
    WHY("keyring is not loaded");
    return NULL;
  }

  if (!pin) pin="";

  keyring_identity *id = keyring_new_identity();
  if (!id)
    goto kci_safeexit;

  /* Remember pin */
  if (pin && *pin && !(id->PKRPin = str_edup(pin)))
    goto kci_safeexit;

  /* Find free slot in keyring. */
  id->slot = find_free_slot(k);
  if (id->slot == 0) {
    WHY("no free slots in first slab (no support for more than one slab)");
    goto kci_safeexit;
  }

  /* Mark slot as occupied and internalise new identity. */
  if (!keyring_commit_identity(k, id))
    goto kci_safeexit;

  /* Everything went fine */
  INFOF("created identity slot=%u SID=%s", id->slot, alloca_tohex_sid_t(id->subscriber->sid));
  k->dirty = 1;
  return id;

 kci_safeexit:
  if (id)
    free_identity(id);
  return NULL;
}

static int write_random_slot(keyring_file *k, unsigned slot)
{
  if (is_slot_allocated(k, slot))
    return 0;

  DEBUGF(keyring, "Fill slot %u with randomness", slot);
  uint8_t random_data[KEYRING_PAGE_SIZE];
  randombytes_buf(random_data, sizeof random_data);

  off_t file_offset = KEYRING_PAGE_SIZE * slot;

  if (fseeko(k->file, k->file_size, SEEK_SET) == -1)
    return WHYF_perror("fseeko(%d, %ld, SEEK_SET)", fileno(k->file), (long)file_offset);
  if (fwrite(random_data, sizeof random_data, 1, k->file) != 1)
    return WHYF_perror("fwrite(%p, %ld, 1, %d)", random_data, sizeof random_data, fileno(k->file));

  if (k->file_size < file_offset + KEYRING_PAGE_SIZE)
      k->file_size = file_offset + KEYRING_PAGE_SIZE;

  return 0;
}

/* Remove the given identity from the keyring by overwriting it's slot in the keyring file with
 * random data, and unlinking it from the in-memory cache list.  Does NOT call
 * free_identity(id), so the identity's contents remain intact; the caller must free the identity if
 * desired.
 */
void keyring_destroy_identity(keyring_file *k, keyring_identity *id)
{
  DEBUGF(keyring, "k=%p, id=%p, id->slot=%u", k, id, id->slot);
  if (!k->bam) {
    WHY("keyring is not loaded");
    return;
  }
  assert(id->slot != 0);

  keyring_identity **i = &k->identities;
  while (*i && *i != id)
    i = &(*i)->next;
  assert(*i); // the identity being released must be in the keyring

  // Mark the slot as unallocated in the BAM and un-loaded in memory.
  mark_slot_allocated(k, id->slot, 0);
  mark_slot_loaded(k, id->slot, 0);

  // Fill the slot in the file with random bytes.
  write_random_slot(k, id->slot);

  // Unlink the identity from the in-memory cache.
  *i = id->next;
  id->next = NULL;
}

int keyring_commit(keyring_file *k)
{
  DEBUGF(keyring, "k=%p", k);
  unsigned errorCount = 0;
  /* Write all BAMs */
  keyring_bam *b;
  for (b = k->bam; b; b = b->next) {
    if (fseeko(k->file, b->file_offset, SEEK_SET) == -1) {
      WHYF_perror("fseeko(%d, %ld, SEEK_SET)", fileno(k->file), (long)b->file_offset);
      errorCount++;
    } else if (fwrite(b->allocmap, KEYRING_BAM_BYTES, 1, k->file) != 1) {
      WHYF_perror("fwrite(%p, %ld, 1, %d)", b->allocmap, (long)KEYRING_BAM_BYTES, fileno(k->file));
      errorCount++;
    } else if (fwrite(k->KeyRingSalt, k->KeyRingSaltLen, 1, k->file)!=1) {
      WHYF_perror("fwrite(%p, %ld, 1, %d)", k->KeyRingSalt, (long)k->KeyRingSaltLen, fileno(k->file));
      errorCount++;
    }
  }
  /* For each identity in each context, write the record to disk.
     This re-salts every identity as it is re-written, and the pin
     for each identity and context is used, so changing a keypair or pin
     is as simple as updating the keyring_identity or related structure,
     and then calling this function. */
  keyring_iterator it;
  keyring_iterator_start(k, &it);
  while(keyring_next_identity(&it)){
    if (it.identity->slot == 0){
      it.identity->slot = find_free_slot(k);
      DEBUGF(keyring, "Allocate identity into slot %u", it.identity->slot);
    }
    unsigned char pkr[KEYRING_PAGE_SIZE];

    if (keyring_pack_identity(it.identity, pkr)){
      errorCount++;
      continue;
    }
    /* Now crypt and store block */
    /* Crypt */
    if (keyring_munge_block(pkr, KEYRING_PAGE_SIZE,
      it.file->KeyRingSalt, it.file->KeyRingSaltLen,
      it.file->KeyRingPin, it.identity->PKRPin)) {
      WHY("keyring_munge_block() failed");
      errorCount++;
      continue;
    }

    /* Store */
    off_t file_offset = KEYRING_PAGE_SIZE * it.identity->slot;

    while ((off_t)k->file_size < file_offset){
      // write randomness into any blank keyring entries
      // ignoring any range we are about to write anyway
      unsigned slot = k->file_size / KEYRING_PAGE_SIZE;
      if (write_random_slot(k, slot)!=0){
	errorCount++;
	break;
      }
    }

    DEBUGF(keyring, "Write identity to slot %u", it.identity->slot);

    if (fseeko(k->file, file_offset, SEEK_SET) == -1) {
      WHYF_perror("fseeko(%d, %ld, SEEK_SET)", fileno(k->file), (long)file_offset);
      errorCount++;
      continue;
    }

    if (fwrite(pkr, KEYRING_PAGE_SIZE, 1, k->file) != 1) {
      WHYF_perror("fwrite(%p, %ld, 1, %d)", pkr, (long)KEYRING_PAGE_SIZE, fileno(k->file));
      errorCount++;
      continue;
    }

    if (k->file_size < file_offset + KEYRING_PAGE_SIZE)
      k->file_size = file_offset + KEYRING_PAGE_SIZE;
  }

  // keep writing random bytes until the number of slots that might be used is an exact multiple of KEYRING_ALLOC_CHUNK
  while(1){
    unsigned slot_count = k->file_size / KEYRING_PAGE_SIZE;
    if ((slot_count % KEYRING_ALLOC_CHUNK) == 0)
      break;
    if (write_random_slot(k, slot_count)!=0)
      break;
  }

  if (fflush(k->file) == -1) {
    WHYF_perror("fflush(%d)", fileno(k->file));
    errorCount++;
  }
  if (!errorCount)
    k->dirty=0;
  return errorCount ? WHYF("%u errors commiting keyring to disk", errorCount) : 0;
}

int keyring_set_did_name(keyring_identity *id, const char *did, const char *name)
{
  /* Do nothing if not changing either field. */
  if (!did && !name)
    return 0;

  /* Find where to put it */
  keypair *kp = id->keypairs;
  while(kp){
    if (kp->type==KEYTYPE_DID){
      DEBUG(keyring, "Identity already contains DID/Name");
      break;
    }
    kp=kp->next;
  }
  
  /* Allocate if not found */
  if (!kp){
    if ((kp = keyring_alloc_keypair(KEYTYPE_DID, 0)) == NULL)
      return -1;
    keyring_identity_add_keypair(id, kp);
    DEBUG(keyring, "Created DID/Name record for identity");
    bzero(kp->private_key, kp->private_key_len);
    bzero(kp->public_key, kp->public_key_len);
  }

  /* Store DID as nul-terminated string. */
  if (did) {
    size_t len = strlen(did);
    assert(len < kp->private_key_len);
    bcopy(did, &kp->private_key[0], len);
    bzero(&kp->private_key[len], kp->private_key_len - len);
    DEBUG_dump(keyring, "storing DID", &kp->private_key[0], kp->private_key_len);
  }

  /* Store Name as nul-terminated string. */
  if (name) {
    size_t len = strlen(name);
    assert(len < kp->public_key_len);
    bcopy(name, &kp->public_key[0], len);
    bzero(&kp->public_key[len], kp->public_key_len - len);
    DEBUG_dump(keyring, "storing Name", &kp->public_key[0], kp->public_key_len);
  }

  return 0;
}

int keyring_set_pin(keyring_identity *id, const char *pin)
{
  if (id->PKRPin){
    free(id->PKRPin);
    id->PKRPin = NULL;
  }
  if (pin && *pin)
    id->PKRPin = str_edup(pin);
  return 0;
}

int keyring_unpack_tag(const unsigned char *packed, size_t packed_len, const char **name, const unsigned char **value, size_t *length)
{
  size_t i;
  for (i=0;i<packed_len;i++){
    if (packed[i]==0){
      *name = (const char*)packed;
      if (value)
	*value = &packed[i+1];
      if (length)
	*length = packed_len - (i+1);
      return 0;
    }
  }
  return WHY("Did not find NULL values in tag");
}

int keyring_pack_tag(unsigned char *packed, size_t *packed_len, const char *name, const unsigned char *value, size_t length)
{
  size_t name_len=strlen(name)+1;
  if (packed && *packed_len <name_len+length)
    return -1;
  *packed_len=name_len+length;
  if (packed){
    bcopy(name, packed, name_len);
    bcopy(value, &packed[name_len], length);
  }
  return 0;
}

int keyring_set_public_tag(keyring_identity *id, const char *name, const unsigned char *value, size_t length)
{
  keypair *kp=id->keypairs;
  while(kp){
    const char *tag_name;
    const unsigned char *tag_value;
    size_t tag_length;
    if (kp->type==KEYTYPE_PUBLIC_TAG &&
      keyring_unpack_tag(kp->public_key, kp->public_key_len, 
	  &tag_name, &tag_value, &tag_length)==0 &&
      strcmp(tag_name, name)==0) {
      DEBUG(keyring, "Found existing public tag");
      break;
    }
    kp = kp->next;
  }
  
  /* allocate if needed */
  if (!kp){
    DEBUGF(keyring, "Creating new public tag");
    if ((kp = keyring_alloc_keypair(KEYTYPE_PUBLIC_TAG, 0)) == NULL)
      return -1;
    keyring_identity_add_keypair(id, kp);
  }
  
  if (kp->public_key)
    free(kp->public_key);
  
  if (keyring_pack_tag(NULL, &kp->public_key_len, name, value, length))
    return -1;
  kp->public_key = emalloc(kp->public_key_len);
  if (!kp->public_key)
    return -1;
  if (keyring_pack_tag(kp->public_key, &kp->public_key_len, name, value, length))
    return -1;
  
  DEBUG_dump(keyring, "New tag", kp->public_key, kp->public_key_len);
  return 0;
}

keypair * keyring_find_public_tag(keyring_iterator *it, const char *name, const unsigned char **value, size_t *length)
{
  keypair *keypair;
  while((keypair=keyring_next_keytype(it,KEYTYPE_PUBLIC_TAG))){
    const char *tag_name;
    if (!keyring_unpack_tag(keypair->public_key, keypair->public_key_len, &tag_name, value, length) &&
      strcmp(name, tag_name)==0){
      return keypair;
    }
  }
  if (value)
    *value=NULL;
  return NULL;
}

keypair * keyring_find_public_tag_value(keyring_iterator *it, const char *name, const unsigned char *value, size_t length)
{
  const unsigned char *stored_value;
  size_t stored_length;
  keypair *keypair;
  while((keypair=keyring_find_public_tag(it, name, &stored_value, &stored_length))){
    if (stored_length == length && memcmp(value, stored_value, length)==0)
      return keypair;
  }
  return NULL;
}

// sign the hash of a message, adding the signature to the end of the message buffer.
int keyring_sign_message(struct keyring_identity *identity, unsigned char *content, size_t buffer_len, size_t *content_len)
{
  if (*content_len + SIGNATURE_BYTES > buffer_len)
    return WHYF("Insufficient space in message buffer to add signature. %zu, need %zu",buffer_len, *content_len + SIGNATURE_BYTES);

  unsigned char hash[crypto_hash_sha512_BYTES];
  crypto_hash_sha512(hash, content, *content_len);

  if (crypto_sign_detached(&content[*content_len], NULL, hash, crypto_hash_sha512_BYTES, identity->sign_keypair->binary))
    return WHY("Signing failed");

  *content_len += SIGNATURE_BYTES;
  return 0;
}

// someone else is claiming to be me on this network
// politely request that they release my identity
int keyring_send_unlock(struct subscriber *subscriber)
{
  if (!subscriber->identity)
    return WHY("Cannot unlock an identity we don't have in our keyring");
  if (subscriber->reachable & REACHABLE_SELF)
    return 0;
    
  struct internal_mdp_header header;
  bzero(&header, sizeof header);
  
  header.source = get_my_subscriber(1);
  header.destination = subscriber;
  header.source_port = MDP_PORT_KEYMAPREQUEST;
  header.destination_port = MDP_PORT_KEYMAPREQUEST;
  header.qos = OQ_MESH_MANAGEMENT;
  
  // use a fixed buffer so we know there's enough space for the signature
  uint8_t buff[MDP_MTU];
  struct overlay_buffer *response = ob_static(buff, sizeof buff);
  ob_append_byte(response, UNLOCK_REQUEST);
  
  size_t len = ob_position(response);
  if (keyring_sign_message(subscriber->identity, ob_ptr(response), sizeof(buff), &len))
    return -1;
    
  ob_append_space(response, len - ob_position(response));
  
  DEBUGF(keyring, "Sending Unlock request for sid %s", alloca_tohex_sid_t(subscriber->sid));
  ob_flip(response);
  int ret = overlay_send_frame(&header, response);
  ob_free(response);
  return ret;
}

int keyring_send_identity_request(struct subscriber *subscriber){
  if (subscriber->id_valid)
    return 0;
  
  time_ms_t now = gettime_ms();
  
  if (now < subscriber->id_last_request + 100){
    DEBUG(keyring, "Too soon to ask for SAS mapping again");
    return 0;
  }
  
  DEBUGF(keyring, "Requesting SAS mapping for SID=%s", alloca_tohex_sid_t(subscriber->sid));
  
  /* request mapping (send request auth-crypted). */
  struct internal_mdp_header header;
  bzero(&header, sizeof header);
  
  header.source = get_my_subscriber(1);
  header.destination = subscriber;
  header.source_port = MDP_PORT_KEYMAPREQUEST;
  header.destination_port = MDP_PORT_KEYMAPREQUEST;
  header.qos = OQ_MESH_MANAGEMENT;
  
  struct overlay_buffer *payload = ob_new();
  ob_append_byte(payload, KEYTYPE_CRYPTOSIGN);
  
  DEBUGF(keyring, "Sending SAS resolution request");
  subscriber->id_last_request=now;
  ob_flip(payload);
  int ret = overlay_send_frame(&header, payload);
  ob_free(payload);
  return ret;
}

void keyring_identity_extract(const keyring_identity *id, const char **didp, const char **namep)
{
  keypair *kp=id->keypairs;
  while(kp){
    switch (kp->type) {
    case KEYTYPE_DID:
      if (didp)
	*didp = (const char *) kp->private_key;
      if (namep)
	*namep = (const char *) kp->public_key;
      return;
    case KEYTYPE_CRYPTOBOX:
    case KEYTYPE_CRYPTOSIGN:
    case KEYTYPE_RHIZOME:
    case KEYTYPE_PUBLIC_TAG:
    case KEYTYPE_CRYPTOCOMBINED:
      break;
    case KEYTYPE_INVALID:
      FATALF("ktype = 0x%02x(%s)", kp->type, keytype_str(kp->type, "unknown"));
    }
    kp=kp->next;
  }
}

keyring_file *keyring_create_instance()
{
  return keyring_open_create_instance("", 1);
}

keyring_file *keyring_open_instance(const char *pin)
{
  return keyring_open_create_instance(pin, 0);
}

static keyring_file *keyring_open_create_instance(const char *pin, int force_create)
{
  keyring_file *k = NULL;
  IN();
  if (create_serval_instance_dir() == -1)
    RETURN(NULL);
  // Work out the absolute path to the keyring file.
  const char *env = getenv("SERVALD_KEYRING_PATH");
  if (!env)
    env = "serval.keyring";
  char keyringFile[1024];
  if (!FORMF_SERVAL_ETC_PATH(keyringFile, "%s", env))
    RETURN(NULL);
  // Work out if the keyring file is writeable.
  const char *readonly_env = getenv("SERVALD_KEYRING_READONLY");
  bool_t readonly_b;
  int writeable = readonly_env == NULL || cf_opt_boolean(&readonly_b, readonly_env) != CFOK || !readonly_b;
  if ((k = keyring_open_or_create(keyringFile, writeable)) == NULL)
    RETURN(NULL);
  if ((force_create || k->file_size < KEYRING_PAGE_SIZE) && keyring_initialise(k) == -1) {
    keyring_free(k);
    return NULL;
  }
  if (keyring_load(k, pin) == -1) {
    keyring_free(k);
    return NULL;
  }
  RETURN(k);
  OUT();
}

keyring_file *keyring_open_instance_cli(const struct cli_parsed *parsed)
{
  IN();
  const char *kpin = NULL;
  cli_arg(parsed, "--keyring-pin", &kpin, NULL, "");
  keyring_file *k = keyring_open_instance(kpin);
  if (k == NULL)
    RETURN(NULL);
  // Always open all PIN-less entries.
  keyring_enter_pin(k, "");
  // Open all entries for which an entry PIN has been given.
  unsigned i;
  for (i = 0; i < parsed->labelc; ++i)
    if (strn_str_cmp(parsed->labelv[i].label, parsed->labelv[i].len, "--entry-pin") == 0)
      keyring_enter_pin(k, parsed->labelv[i].text);
  RETURN(k);
  OUT();
}

/*
  The CryptoBox function of NaCl involves a scalar mult operation between the
  public key of the recipient and the private key of the sender (or vice versa).
  This can take about 1 cpu second on a phone, which is rather bad.
  Fortunately, NaCl allows the caching of the result of this computation, which can
  then be fed into the process to make it much, much faster.
  Thus we need a mechanism for caching the various scalarmult results so that they
  can indeed be reused.
*/

/* XXX We need a more efficient implementation than a linear list, but it will
   do for now. */
struct nm_record {
  /* 96 bytes per record */
  sid_t known_key;
  sid_t unknown_key;
  unsigned char nm_bytes[crypto_box_BEFORENMBYTES];
};

unsigned nm_slots_used=0;
/* 512 x 96 bytes = 48KB, not too big */
#define NM_CACHE_SLOTS 512
struct nm_record nm_cache[NM_CACHE_SLOTS];

unsigned char *keyring_get_nm_bytes(const uint8_t *box_sk, const sid_t *box_pk, const sid_t *unknown_sidp)
{
  IN();

  /* See if we have it cached already */
  unsigned i;
  for(i=0;i<nm_slots_used;i++){
    if (cmp_sid_t(&nm_cache[i].known_key, box_pk) != 0) continue;
    if (cmp_sid_t(&nm_cache[i].unknown_key, unknown_sidp) != 0) continue;
    RETURN(nm_cache[i].nm_bytes);
  }

  /* Not in the cache, so prepare to cache it (or return failure if known is not
     in fact a known key */
  /* work out where to store it */
  if (nm_slots_used<NM_CACHE_SLOTS) {
    i=nm_slots_used; nm_slots_used++; 
  } else {
    i=randombytes_uniform(NM_CACHE_SLOTS);
  }

  /* calculate and store */
  nm_cache[i].known_key = *box_pk;
  nm_cache[i].unknown_key = *unknown_sidp;
  if (crypto_box_beforenm(nm_cache[i].nm_bytes, unknown_sidp->binary, box_sk)){
    WHY("crypto_box_beforenm failed");
    RETURN(NULL);
  }
  RETURN(nm_cache[i].nm_bytes);
  OUT();
}

static int cmp_identity_ptrs(const keyring_identity *const *a, const keyring_identity *const *b)
{
  if (a==b)
    return 0;
  
  keypair *kpa=(*a)->keypairs, *kpb=(*b)->keypairs;
  int c;
  while(kpa && kpb){
    if ((c = cmp_keypair(kpa, kpb)))
      return c;
    kpa=kpa->next;
    kpb=kpb->next;
  }
  
  if (kpa)
    return 1;
  if (kpb)
    return -1;
  return 0;
}

static void keyring_dump_keypair(const keypair *kp, XPRINTF xpf, int include_secret)
{
  assert(kp->type != 0);
  assert(kp->type < NELS(key_types));
  xprintf(xpf, "type=0x%02x(%s) ", kp->type, keytype_str(kp->type, "unknown"));
  if (key_types[kp->type].dumper)
    key_types[kp->type].dumper(kp, xpf, include_secret);
  else
    dump_private_public(kp, xpf, include_secret);
}

int keyring_dump(keyring_file *k, XPRINTF xpf, int include_secret)
{
  unsigned nids = 0;
  
  keyring_iterator it;
  keyring_iterator_start(k, &it);
  while(keyring_next_identity(&it))
    ++nids;
  
  unsigned i = 0;
  const keyring_identity *idx[nids];
  
  keyring_iterator_start(k, &it);
  while(keyring_next_identity(&it)){
    assert(i < nids);
    idx[i++] = it.identity;
  }
  assert(i == nids);
  
  qsort(idx, nids, sizeof(idx[0]), (int(*)(const void *, const void *)) cmp_identity_ptrs);
  for (i = 0; i != nids; ++i) {
    keypair *kp=idx[i]->keypairs;
    while(kp){
      xprintf(xpf, "%u: ", i);
      keyring_dump_keypair(kp, xpf, include_secret);
      xprintf(xpf, "\n");
      kp=kp->next;
    }
  }
  return 0;
}

int keyring_load_from_dump(keyring_file *k, unsigned entry_pinc, const char **entry_pinv, FILE *input)
{
  clearerr(input);
  char line[1024];
  unsigned pini = 0;
  keyring_identity *id = NULL;
  unsigned last_idn = 0;
  while (fgets(line, sizeof line - 1, input) != NULL) {
    // Strip trailing \n or CRLF
    size_t linelen = strlen(line);
    if (linelen && line[linelen - 1] == '\n') {
      line[--linelen] = '\0';
      if (linelen && line[linelen - 1] == '\r')
	line[--linelen] = '\0';
    } else {
      if (id)
	free_identity(id);
      return WHY("line too long");
    }
    unsigned idn;
    unsigned ktype;
    int i, j;
    int n = sscanf(line, "%u: type=0x%x (%n%*[^)]%n)", &idn, &ktype, &i, &j);
    if (n == EOF && (ferror(input) || feof(input)))
	break;
    if (n != 2){
      if (id)
	free_identity(id);
      return WHYF("malformed input n=%u", n);
    }
    if (ktype == KEYTYPE_INVALID){
      if (id)
	free_identity(id);
      return WHY("invalid input: ktype=0");
    }
    const char *ktypestr = &line[i];
    line[j] = '\0';
    const char *content = &line[j + 1];
    //DEBUGF(keyring, "n=%d i=%u ktypestr=%s j=%u content=%s", n, i, alloca_str_toprint(ktypestr), j, alloca_str_toprint(content));
    keypair *kp = keyring_alloc_keypair(ktype, 0);
    if (kp == NULL){
      if (id)
	free_identity(id);
      return -1;
    }
    int (*loader)(keypair *, const char *) = load_unknown;
    if (strcmp(ktypestr, "unknown") != 0 && ktype < NELS(key_types))
      loader = key_types[ktype].loader;
    if (loader(kp, content) == -1) {
      if (id)
	free_identity(id);
      keyring_free_keypair(kp);
      return -1;
    }
    if (id == NULL || idn != last_idn) {
      last_idn = idn;
      if (id){
	if (!keyring_commit_identity(k, id))
	  free_identity(id);
	else
	  k->dirty=1;
      }
      if ((id = emalloc_zero(sizeof(keyring_identity))) == NULL) {
	keyring_free_keypair(kp);
	return -1;
      }
      if (pini < entry_pinc  && (id->PKRPin = str_edup(entry_pinv[pini++])) == NULL) {
	keyring_free_keypair(kp);
	free_identity(id);
	return -1;
      }
      if ((id->slot = find_free_slot(k)) == 0) {
	keyring_free_keypair(kp);
	free_identity(id);
	return WHY("no free slot");
      }
      DEBUGF(keyring, "Allocate identity into slot %u", id->slot);
    }
    if (!keyring_identity_add_keypair(id, kp))
      keyring_free_keypair(kp);
  }
  if (id){
    if (!keyring_commit_identity(k, id))
      free_identity(id);
    else
      k->dirty=1;
  }
  if (ferror(input))
    return WHYF_perror("fscanf");
  return 0;
}

/* Free the global keyring after every CLI command.
 */

__thread keyring_file *keyring = NULL;

static void keyring_on_cmd_cleanup();
DEFINE_TRIGGER(cmd_cleanup, keyring_on_cmd_cleanup);
static void keyring_on_cmd_cleanup()
{
  keyring_free(keyring);
  keyring = NULL;
  free_subscribers();
}