git.sur5r.net Git - openldap/blob - libraries/liblmdb/mdb_load.c

   1 /* mdb_load.c - memory-mapped database load tool */
   2 /*
   3  * Copyright 2011-2014 Howard Chu, Symas Corp.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted only as authorized by the OpenLDAP
   8  * Public License.
   9  *
  10  * A copy of this license is available in the file LICENSE in the
  11  * top-level directory of the distribution or, alternatively, at
  12  * <http://www.OpenLDAP.org/license.html>.
  13  */
  14 #include <stdio.h>
  15 #include <stdlib.h>
  16 #include <errno.h>
  17 #include <string.h>
  18 #include <ctype.h>
  19 #include <unistd.h>
  20 #include "lmdb.h"
  21
  22 #define PRINT   1
  23 #define NOHDR   2
  24 static int mode;
  25
  26 static char *subname = NULL;
  27
  28 static size_t lineno;
  29 static int version;
  30
  31 static int flags;
  32
  33 static char *prog;
  34
  35 static int eof;
  36
  37 static MDB_val kbuf, dbuf;
  38
  39 #define STRLENOF(s)     (sizeof(s)-1)
  40
  41 typedef struct flagbit {
  42         int bit;
  43         char *name;
  44         int len;
  45 } flagbit;
  46
  47 #define S(s)    s, STRLENOF(s)
  48
  49 flagbit dbflags[] = {
  50         { MDB_REVERSEKEY, S("reversekey") },
  51         { MDB_DUPSORT, S("dupsort") },
  52         { MDB_INTEGERKEY, S("integerkey") },
  53         { MDB_DUPFIXED, S("dupfixed") },
  54         { MDB_INTEGERDUP, S("integerdup") },
  55         { MDB_REVERSEDUP, S("reversedup") },
  56         { 0, NULL, 0 }
  57 };
  58
  59 static const char hexc[] = "0123456789abcdef";
  60
  61 static void readhdr()
  62 {
  63         char *ptr;
  64
  65         while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) {
  66                 lineno++;
  67                 if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
  68                         version=atoi(dbuf.mv_data+STRLENOF("VERSION="));
  69                         if (version > 3) {
  70                                 fprintf(stderr, "%s: line %zd: unsupported VERSION %d\n",
  71                                         prog, lineno, version);
  72                                 exit(EXIT_FAILURE);
  73                         }
  74                 } else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) {
  75                         break;
  76                 } else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) {
  77                         if (!strncmp(dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print")))
  78                                 mode |= PRINT;
  79                         else if (strncmp(dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) {
  80                                 fprintf(stderr, "%s: line %zd: unsupported FORMAT %s\n",
  81                                         prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT="));
  82                                 exit(EXIT_FAILURE);
  83                         }
  84                 } else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) {
  85                         ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
  86                         if (ptr) *ptr = '\0';
  87                         if (subname) free(subname);
  88                         subname = strdup(dbuf.mv_data+STRLENOF("database="));
  89                 } else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) {
  90                         if (strncmp(dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree")))  {
  91                                 fprintf(stderr, "%s: line %zd: unsupported type %s\n",
  92                                         prog, lineno, (char *)dbuf.mv_data+STRLENOF("type="));
  93                                 exit(EXIT_FAILURE);
  94                         }
  95                 } else {
  96                         int i;
  97                         for (i=0; dbflags[i].bit; i++) {
  98                                 if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) &&
  99                                         ((char *)dbuf.mv_data)[dbflags[i].len] == '=') {
 100                                         flags |= dbflags[i].bit;
 101                                         break;
 102                                 }
 103                         }
 104                         if (!dbflags[i].bit) {
 105                                 ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size);
 106                                 if (!ptr) {
 107                                         fprintf(stderr, "%s: line %zd: unexpected format\n",
 108                                                 prog, lineno);
 109                                         exit(EXIT_FAILURE);
 110                                 } else {
 111                                         *ptr = '\0';
 112                                         fprintf(stderr, "%s: line %zd: unrecognized keyword ignored: %s\n",
 113                                                 prog, lineno, (char *)dbuf.mv_data);
 114                                 }
 115                         }
 116                 }
 117         }
 118 }
 119
 120 static void badend()
 121 {
 122         fprintf(stderr, "%s: line %zd: unexpected end of input\n",
 123                 prog, lineno);
 124 }
 125
 126 static int unhex(unsigned char *c2)
 127 {
 128         int x, c;
 129         x = *c2++ & 0x4f;
 130         if (x & 0x40)
 131                 x -= 55;
 132         c = x << 4;
 133         x = *c2 & 0x4f;
 134         if (x & 0x40)
 135                 x -= 55;
 136         c |= x;
 137         return c;
 138 }
 139
 140 static int readline(MDB_val *out, MDB_val *buf)
 141 {
 142         unsigned char *c1, *c2, *end;
 143         size_t len;
 144         int c;
 145
 146         if (!(mode & NOHDR)) {
 147                 c = fgetc(stdin);
 148                 if (c == EOF) {
 149                         eof = 1;
 150                         return EOF;
 151                 }
 152                 if (c != ' ') {
 153                         if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
 154 badend:
 155                                 eof = 1;
 156                                 badend();
 157                                 return EOF;
 158                         }
 159                         if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END")))
 160                                 return EOF;
 161                         goto badend;
 162                 }
 163         }
 164         if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
 165                 eof = 1;
 166                 return EOF;
 167         }
 168         lineno++;
 169
 170         c1 = buf->mv_data;
 171         len = strlen((char *)c1);
 172
 173         /* Is buffer too short? */
 174         while (c1[len-1] != '\n') {
 175                 buf->mv_data = realloc(buf->mv_data, buf->mv_size*2);
 176                 if (!buf->mv_data) {
 177                         eof = 1;
 178                         fprintf(stderr, "%s: line %zd: out of memory, line too long\n",
 179                                 prog, lineno);
 180                         return EOF;
 181                 }
 182                 c1 = buf->mv_data;
 183                 c1 += buf->mv_size;
 184                 if (fgets((char *)c1, buf->mv_size, stdin) == NULL) {
 185                         eof = 1;
 186                         badend();
 187                         return EOF;
 188                 }
 189                 buf->mv_size *= 2;
 190                 len = strlen((char *)c1);
 191         }
 192         c1 = c2 = buf->mv_data;
 193         len = strlen((char *)c1);
 194         c1[--len] = '\0';
 195         end = c1 + len;
 196
 197         if (mode & PRINT) {
 198                 while (c2 < end) {
 199                         if (*c2 == '\\') {
 200                                 if (c2[1] == '\\') {
 201                                         c1++; c2 += 2;
 202                                 } else {
 203                                         if (c2+3 >= end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
 204                                                 eof = 1;
 205                                                 badend();
 206                                                 return EOF;
 207                                         }
 208                                         *c1++ = unhex(++c2);
 209                                         c2 += 2;
 210                                 }
 211                         } else {
 212                                 c1++; c2++;
 213                         }
 214                 }
 215         } else {
 216                 /* odd length not allowed */
 217                 if (len & 1) {
 218                         eof = 1;
 219                         badend();
 220                         return EOF;
 221                 }
 222                 while (c2 < end) {
 223                         if (!isxdigit(*c2) || !isxdigit(c2[1])) {
 224                                 eof = 1;
 225                                 badend();
 226                                 return EOF;
 227                         }
 228                         *c1++ = unhex(c2);
 229                         c2 += 2;
 230                 }
 231         }
 232         c2 = out->mv_data = buf->mv_data;
 233         out->mv_size = c1 - c2;
 234
 235         return 0;
 236 }
 237
 238 static void usage()
 239 {
 240         fprintf(stderr, "usage: %s dbpath [-V] [-f input] [-n] [-s name] [-N] [-T]\n", prog);
 241         exit(EXIT_FAILURE);
 242 }
 243
 244 int main(int argc, char *argv[])
 245 {
 246         int i, rc;
 247         MDB_env *env;
 248         MDB_txn *txn;
 249         MDB_cursor *mc;
 250         MDB_dbi dbi;
 251         char *envname;
 252         int envflags = 0, putflags = 0;
 253
 254         prog = argv[0];
 255
 256         if (argc < 2) {
 257                 usage(prog);
 258         }
 259
 260         /* -f: load file instead of stdin
 261          * -n: use NOSUBDIR flag on env_open
 262          * -s: load into named subDB
 263          * -N: use NOOVERWRITE on puts
 264          * -T: read plaintext
 265          * -V: print version and exit
 266          */
 267         while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
 268                 switch(i) {
 269                 case 'V':
 270                         printf("%s\n", MDB_VERSION_STRING);
 271                         exit(0);
 272                         break;
 273                 case 'f':
 274                         if (freopen(optarg, "r", stdin) == NULL) {
 275                                 fprintf(stderr, "%s: %s: reopen: %s\n",
 276                                         prog, optarg, strerror(errno));
 277                                 exit(EXIT_FAILURE);
 278                         }
 279                         break;
 280                 case 'n':
 281                         envflags |= MDB_NOSUBDIR;
 282                         break;
 283                 case 's':
 284                         subname = strdup(optarg);
 285                         break;
 286                 case 'N':
 287                         putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
 288                         break;
 289                 case 'T':
 290                         mode |= NOHDR;
 291                         break;
 292                 default:
 293                         usage(prog);
 294                 }
 295         }
 296
 297         if (optind != argc - 1)
 298                 usage(prog);
 299
 300         envname = argv[optind];
 301         rc = mdb_env_create(&env);
 302
 303         if (subname) {
 304                 mdb_env_set_maxdbs(env, 2);
 305         }
 306
 307         rc = mdb_env_open(env, envname, envflags, 0664);
 308         if (rc) {
 309                 printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
 310                 goto env_close;
 311         }
 312
 313         kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
 314         kbuf.mv_data = malloc(kbuf.mv_size);
 315         dbuf.mv_size = 4096;
 316         dbuf.mv_data = malloc(dbuf.mv_size);
 317
 318         while(!eof) {
 319                 MDB_val key, data;
 320                 int batch = 0;
 321                 flags = 0;
 322
 323                 if (!(mode & NOHDR))
 324                         readhdr();
 325
 326                 rc = mdb_txn_begin(env, NULL, 0, &txn);
 327                 if (rc) {
 328                         printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
 329                         goto env_close;
 330                 }
 331
 332                 rc = mdb_open(txn, subname, flags, &dbi);
 333                 if (rc) {
 334                         printf("mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
 335                         goto txn_abort;
 336                 }
 337
 338                 rc = mdb_cursor_open(txn, dbi, &mc);
 339                 if (rc) {
 340                         printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
 341                         goto txn_abort;
 342                 }
 343
 344                 while(1) {
 345                         rc = readline(&key, &kbuf);
 346                         if (rc == EOF)
 347                                 break;
 348                         if (rc)
 349                                 goto txn_abort;
 350
 351                         rc = readline(&data, &dbuf);
 352                         if (rc)
 353                                 goto txn_abort;
 354
 355                         rc = mdb_cursor_put(mc, &key, &data, putflags);
 356                         if (rc == MDB_KEYEXIST && putflags)
 357                                 continue;
 358                         if (rc)
 359                                 goto txn_abort;
 360                         batch++;
 361                         if (batch == 100) {
 362                                 rc = mdb_txn_commit(txn);
 363                                 if (rc) {
 364                                         fprintf(stderr, "%s: line %zd: txn_commit: %s\n",
 365                                                 prog, lineno, mdb_strerror(rc));
 366                                         goto env_close;
 367                                 }
 368                                 rc = mdb_txn_begin(env, NULL, 0, &txn);
 369                                 if (rc) {
 370                                         printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
 371                                         goto env_close;
 372                                 }
 373                                 rc = mdb_cursor_open(txn, dbi, &mc);
 374                                 if (rc) {
 375                                         printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
 376                                         goto txn_abort;
 377                                 }
 378                                 batch = 0;
 379                         }
 380                 }
 381                 rc = mdb_txn_commit(txn);
 382                 txn = NULL;
 383                 if (rc) {
 384                         fprintf(stderr, "%s: line %zd: txn_commit: %s\n",
 385                                 prog, lineno, mdb_strerror(rc));
 386                         goto env_close;
 387                 }
 388                 mdb_dbi_close(env, dbi);
 389         }
 390
 391 txn_abort:
 392         mdb_txn_abort(txn);
 393 env_close:
 394         mdb_env_close(env);
 395
 396         return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 397 }