Symas Lightning Memory-Mapped Database (LMDB) Notes

#include <stdio.h>
#include "lmdb.h"

int main(int argc,char * argv[])
{
  int rc;
  MDB_env *env;
  MDB_dbi dbi;
  MDB_val key, data;
  MDB_txn *txn;
  MDB_cursor *cursor;
  char sval[32];

  /* Note: Most error checking omitted for simplicity */
  rc = mdb_env_create(&env);
  rc = mdb_env_open(env, "./testdb", 0, 0664);
  rc = mdb_txn_begin(env, NULL, 0, &txn);
  rc = mdb_dbi_open(txn, NULL, 0, &dbi);
  key.mv_size = sizeof(int);
  key.mv_data = sval;
  data.mv_size = sizeof(sval);
  data.mv_data = sval;

  sprintf(sval, "%03x %d foo bar", 32, 3141592);
  rc = mdb_put(txn, dbi, &key, &data, 0);
  rc = mdb_txn_commit(txn);
  if (rc) {
    fprintf(stderr, "mdb_txn_commit: (%d) %s\n", rc, mdb_strerror(rc));
    goto leave;
  }
  rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
  rc = mdb_cursor_open(txn, dbi, &cursor);
  while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
    printf("key: %p %.*s, data: %p %.*s\n",
           key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
           data.mv_data, (int) data.mv_size, (char *) data.mv_data);
  }
  mdb_cursor_close(cursor);
  mdb_txn_abort(txn);
leave:
  mdb_dbi_close(env, dbi);
  mdb_env_close(env);
  return 0;
}

mkdir testdb
./sample

key: 0x7f1435f678a4 020 , data: 0x7f1435f678a8 020 3141592 foo bar
key: 0x7f1435f67fdc 1bd , data: 0x7f1435f67fe0 1bd 445 foo bar
key: 0x7f1435f67fb0 1dc , data: 0x7f1435f67fb4 1dc 476 foo bar
key: 0x7f1435f67f84 1e2 , data: 0x7f1435f67f88 1e2 482 foo bar
key: 0x7f1435f67f58 1f9 , data: 0x7f1435f67f5c 1f9 505 foo bar
key: 0x7f1435f67f2c 1ff , data: 0x7f1435f67f30 1ff 511 foo bar

struct MDB_env {
        HANDLE          me_fd;          /**< The main data file */
        HANDLE          me_lfd;         /**< The lock file */
        HANDLE          me_mfd;                 /**< just for writing the meta pages */
        uint32_t        me_flags;               /**< @ref mdb_env */
        unsigned int    me_psize;       /**< DB page size, inited from me_os_psize */
        unsigned int    me_os_psize;    /**< OS page size, from #GET_PAGESIZE */
        unsigned int    me_maxreaders;  /**< size of the reader table */
        /** Max #MDB_txninfo.%mti_numreaders of interest to #mdb_env_close() */
        volatile int    me_close_readers;
        MDB_dbi         me_numdbs;              /**< number of DBs opened */
        MDB_dbi         me_maxdbs;              /**< size of the DB table */
        MDB_PID_T       me_pid;         /**< process ID of this env */
        char            *me_path;               /**< path to the DB files */
        char            *me_map;                /**< the memory map of the data file */
        MDB_txninfo     *me_txns;               /**< the memory map of the lock file or NULL */
        MDB_meta        *me_metas[NUM_METAS];   /**< pointers to the two meta pages */
        void            *me_pbuf;               /**< scratch area for DUPSORT put() */
        MDB_txn         *me_txn;                /**< current write transaction */
        MDB_txn         *me_txn0;               /**< prealloc'd write transaction */
        mdb_size_t      me_mapsize;             /**< size of the data memory map */
        off_t           me_size;                /**< current file size */
        pgno_t          me_maxpg;               /**< me_mapsize / me_psize */
        MDB_dbx         *me_dbxs;               /**< array of static DB info */
        uint16_t        *me_dbflags;    /**< array of flags from MDB_db.md_flags */
        unsigned int    *me_dbiseqs;    /**< array of dbi sequence numbers */
        pthread_key_t   me_txkey;       /**< thread-key for readers */
        txnid_t         me_pgoldest;    /**< ID of oldest reader last time we looked */
        MDB_pgstate     me_pgstate;             /**< state of old pages from freeDB */
#       define          me_pglast       me_pgstate.mf_pglast
#       define          me_pghead       me_pgstate.mf_pghead
        MDB_page        *me_dpages;             /**< list of malloc'd blocks for re-use */
        /** IDL of pages that became unused in a write txn */
        MDB_IDL         me_free_pgs;
        /** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE. */
        MDB_ID2L        me_dirty_list;
        /** Max number of freelist items that can fit in a single overflow page */
        int                     me_maxfree_1pg;
        /** Max size of a node on a page */
        unsigned int    me_nodemax;
#if !(MDB_MAXKEYSIZE)
        unsigned int    me_maxkey;      /**< max size of a key */
#endif
        int             me_live_reader;         /**< have liveness lock in reader table */
#       define          me_rmutex       me_txns->mti_rmutex /**< Shared reader lock */
#       define          me_wmutex       me_txns->mti_wmutex /**< Shared writer lock */
        void            *me_userctx;     /**< User-settable context */
        MDB_assert_func *me_assert_func; /**< Callback for assertion failures */
};

typedef struct MDB_envinfo {
        void    *me_mapaddr;                    /**< Address of map, if fixed */
        mdb_size_t      me_mapsize;                             /**< Size of the data memory map */
        mdb_size_t      me_last_pgno;                   /**< ID of the last used page */
        mdb_size_t      me_last_txnid;                  /**< ID of the last committed transaction */
        unsigned int me_maxreaders;             /**< max reader slots in the environment */
        unsigned int me_numreaders;             /**< max reader slots used in the environment */
} MDB_envinfo;

        /** Meta page content.
         *      A meta page is the start point for accessing a database snapshot.
         *      Pages 0-1 are meta pages. Transaction N writes meta page #(N % 2).
         */
typedef struct MDB_meta {
                /** Stamp identifying this as an LMDB file. It must be set
                 *      to #MDB_MAGIC. */
        uint32_t        mm_magic;
                /** Version number of this file. Must be set to #MDB_DATA_VERSION. */
        uint32_t        mm_version;
        void            *mm_address;            /**< address for fixed mapping */
        pgno_t          mm_mapsize;                     /**< size of mmap region */
        MDB_db          mm_dbs[CORE_DBS];       /**< first is free space, 2nd is main db */
        /** The size of pages used in this DB */
#define mm_psize        mm_dbs[FREE_DBI].md_pad
        /** Any persistent environment flags. @ref mdb_env */
#define mm_flags        mm_dbs[FREE_DBI].md_flags
        pgno_t          mm_last_pg;                     /**< last used page in file */
        volatile txnid_t        mm_txnid;       /**< txnid that committed this page */
} MDB_meta;

typedef struct MDB_page {
#define mp_pgno mp_p.p_pgno
#define mp_next mp_p.p_next
        union {
                pgno_t          p_pgno; /**< page number */
                struct MDB_page *p_next; /**< for in-memory list of freed pages */
        } mp_p;
        uint16_t        mp_pad;
        uint16_t        mp_flags;               /**< @ref mdb_page */
#define mp_lower        mp_pb.pb.pb_lower
#define mp_upper        mp_pb.pb.pb_upper
#define mp_pages        mp_pb.pb_pages
        union {
                struct {
                        indx_t          pb_lower;               /**< lower bound of free space */
                        indx_t          pb_upper;               /**< upper bound of free space */
                } pb;
                uint32_t        pb_pages;       /**< number of overflow pages */
        } mp_pb;
        indx_t          mp_ptrs[1];             /**< dynamic size */
} MDB_page;

typedef struct MDB_node {
        /** lo and hi are used for data size on leaf nodes and for
         * child pgno on branch nodes. On 64 bit platforms, flags
         * is also used for pgno. (Branch nodes have no flags).
         * They are in host byte order in case that lets some
         * accesses be optimized into a 32-bit word access.
         */
        unsigned short  mn_lo, mn_hi;   /**< part of data size or pgno */
        unsigned short  mn_flags;               /**< @ref mdb_node */
        unsigned short  mn_ksize;               /**< key size */
        char            mn_data[1];                     /**< key and data are appended here */
} MDB_node;

/** Information about a single database in the environment. */
typedef struct MDB_db {
        uint32_t        md_pad;         /**< also ksize for LEAF2 pages */
        uint16_t        md_flags;       /**< @ref mdb_dbi_open */
        uint16_t        md_depth;       /**< depth of this tree */
        pgno_t          md_branch_pages;        /**< number of internal pages */
        pgno_t          md_leaf_pages;          /**< number of leaf pages */
        pgno_t          md_overflow_pages;      /**< number of overflow pages */
        mdb_size_t      md_entries;             /**< number of data items */
        pgno_t          md_root;                /**< the root page of this tree */
} MDB_db;

struct MDB_txn {
        MDB_txn         *mt_parent;             /**< parent of a nested txn */
        /** Nested txn under this txn, set together with flag #MDB_TXN_HAS_CHILD */
        MDB_txn         *mt_child;
        pgno_t          mt_next_pgno;   /**< next unallocated page */
        txnid_t         mt_txnid;
        MDB_env         *mt_env;                /**< the DB environment */
        /** The list of pages that became unused during this transaction.
         */
        MDB_IDL         mt_free_pgs;
        /** The list of loose pages that became unused and may be reused
         *      in this transaction, linked through #NEXT_LOOSE_PAGE(page).
         */
        MDB_page        *mt_loose_pgs;
        /* #Number of loose pages (#mt_loose_pgs) */
        int                     mt_loose_count;
        /** The sorted list of dirty pages we temporarily wrote to disk
         *      because the dirty list was full. page numbers in here are
         *      shifted left by 1, deleted slots have the LSB set.
         */
        MDB_IDL         mt_spill_pgs;
        union {
                /** For write txns: Modified pages. Sorted when not MDB_WRITEMAP. */
                MDB_ID2L        dirty_list;
                /** For read txns: This thread/txn's reader table slot, or NULL. */
                MDB_reader      *reader;
        } mt_u;
        /** Array of records for each DB known in the environment. */
        MDB_dbx         *mt_dbxs;
        /** Array of MDB_db records for each known DB */
        MDB_db          *mt_dbs;
        /** Array of sequence numbers for each DB handle */
        unsigned int    *mt_dbiseqs;
        /** In write txns, array of cursors for each DB */
        MDB_cursor      **mt_cursors;
        /** Array of flags for each DB */
        unsigned char   *mt_dbflags;
        /**     Number of DB records in use, or 0 when the txn is finished.
         *      This number only ever increments until the txn finishes; we
         *      don't decrement it when individual DB handles are closed.
         */
        MDB_dbi         mt_numdbs;
        unsigned int    mt_flags;               /**< @ref mdb_txn */
        /** #dirty_list room: Array size - \#dirty pages visible to this txn.
         *      Includes ancestor txns' dirty pages not hidden by other txns'
         *      dirty/spilled pages. Thus commit(nested txn) has room to merge
         *      dirty_list into mt_parent after freeing hidden mt_parent pages.
         */
        unsigned int    mt_dirty_room;
};

struct MDB_cursor {
        /** Next cursor on this DB in this txn */
        MDB_cursor      *mc_next;
        /** Backup of the original cursor if this cursor is a shadow */
        MDB_cursor      *mc_backup;
        /** Context used for databases with #MDB_DUPSORT, otherwise NULL */
        struct MDB_xcursor      *mc_xcursor;
        /** The transaction that owns this cursor */
        MDB_txn         *mc_txn;
        /** The database handle this cursor operates on */
        MDB_dbi         mc_dbi;
        /** The database record for this cursor */
        MDB_db          *mc_db;
        /** The database auxiliary record for this cursor */
        MDB_dbx         *mc_dbx;
        /** The @ref mt_dbflag for this database */
        unsigned char   *mc_dbflag;
        unsigned short  mc_snum;        /**< number of pushed pages */
        unsigned short  mc_top;         /**< index of top page, normally mc_snum-1 */
        unsigned int    mc_flags;       /**< @ref mdb_cursor */
        MDB_page        *mc_pg[CURSOR_STACK];   /**< stack of pushed pages */
        indx_t          mc_ki[CURSOR_STACK];    /**< stack of page indices */
};