git.sur5r.net Git - openldap/blob - libraries/liblmdb/mdb_load.c

   1 /* mdb_load.c - memory-mapped database load tool */
   2 /*
   3  * Copyright 2011-2017 Howard Chu, Symas Corp.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted only as authorized by the OpenLDAP
   8  * Public License.
   9  *
  10  * A copy of this license is available in the file LICENSE in the
  11  * top-level directory of the distribution or, alternatively, at
  12  * <http://www.OpenLDAP.org/license.html>.
  13  */
  14 #include <stdio.h>
  15 #include <stdlib.h>
  16 #include <errno.h>
  17 #include <string.h>
  18 #include <ctype.h>
  19 #include <unistd.h>
  20 #include "lmdb.h"
  21
  22 #define PRINT   1
  23 #define NOHDR   2
  24 static int mode;
  25
  26 static char *subname = NULL;
  27
  28 static size_t lineno;
  29 static int version;
  30
  31 static int flags;
  32
  33 static char *prog;
  34
  35 static int Eof;
  36
  37 static MDB_envinfo info;
  38
  39 static MDB_val kbuf, dbuf;
  40
  41 #ifdef _WIN32
  42 #define Z       "I"
  43 #else
  44 #define Z       "z"
  45 #endif
  46
  47 #define STRLENOF(s)     (sizeof(s)-1)
  48
  49 typedef struct flagbit {
  50         int bit;
  51         char *name;
  52         int len;
  53 } flagbit;
  54
  55 #define S(s)    s, STRLENOF(s)
  56
  57 flagbit dbflags[] = {
  58         { MDB_REVERSEKEY, S("reversekey") },
  59         { MDB_DUPSORT, S("dupsort") },
  60         { MDB_INTEGERKEY, S("integerkey") },
  61         { MDB_DUPFIXED, S("dupfixed") },
  62         { MDB_INTEGERDUP, S("integerdup") },
  63         { MDB_REVERSEDUP, S("reversedup") },
  64         { 0, NULL, 0 }
  65 };
  66
  67 static void readhdr(void)
  68 {
  69         char *ptr;
  70
  71         while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) {
  72                 lineno++;
  73                 if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
  74                         version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION="));
  75                         if (version > 3) {
  76                                 fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n",
  77                                         prog, lineno, version);
  78                                 exit(EXIT_FAILURE);
  79                         }
  80                 } else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) {
  81                         break;
  82                 } else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) {
  83                         if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print")))
  84                                 mode |= PRINT;
  85                         else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) {
  86                                 fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n",
  87                                         prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT="));
  88                                 exit(EXIT_FAILURE);
  89                         }
  90                 } else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) {
  91                         ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
  92                         if (ptr) *ptr = '\0';
  93                         if (subname) free(subname);
  94                         subname = strdup((char *)dbuf.mv_data+STRLENOF("database="));
  95                 } else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) {
  96                         if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree")))  {
  97                                 fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n",
  98                                         prog, lineno, (char *)dbuf.mv_data+STRLENOF("type="));
  99                                 exit(EXIT_FAILURE);
 100                         }
 101                 } else if (!strncmp(dbuf.mv_data, "mapaddr=", STRLENOF("mapaddr="))) {
 102                         int i;
 103                         ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
 104                         if (ptr) *ptr = '\0';
 105                         i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr);
 106                         if (i != 1) {
 107                                 fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n",
 108                                         prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr="));
 109                                 exit(EXIT_FAILURE);
 110                         }
 111                 } else if (!strncmp(dbuf.mv_data, "mapsize=", STRLENOF("mapsize="))) {
 112                         int i;
 113                         ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
 114                         if (ptr) *ptr = '\0';
 115                         i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize);
 116                         if (i != 1) {
 117                                 fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n",
 118                                         prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
 119                                 exit(EXIT_FAILURE);
 120                         }
 121                 } else if (!strncmp(dbuf.mv_data, "maxreaders=", STRLENOF("maxreaders="))) {
 122                         int i;
 123                         ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
 124                         if (ptr) *ptr = '\0';
 125                         i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders);
 126                         if (i != 1) {
 127                                 fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n",
 128                                         prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders="));
 129                                 exit(EXIT_FAILURE);
 130                         }
 131                 } else {
 132                         int i;
 133                         for (i=0; dbflags[i].bit; i++) {
 134                                 if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) &&
 135                                         ((char *)dbuf.mv_data)[dbflags[i].len] == '=') {
 136                                         flags |= dbflags[i].bit;
 137                                         break;
 138                                 }
 139                         }
 140                         if (!dbflags[i].bit) {
 141                                 ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size);
 142                                 if (!ptr) {
 143                                         fprintf(stderr, "%s: line %" Z "d: unexpected format\n",
 144                                                 prog, lineno);
 145                                         exit(EXIT_FAILURE);
 146                                 } else {
 147                                         *ptr = '\0';
 148                                         fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n",
 149                                                 prog, lineno, (char *)dbuf.mv_data);
 150                                 }
 151                         }
 152                 }
 153         }
 154 }
 155
 156 static void badend(void)
 157 {
 158         fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n",
 159                 prog, lineno);
 160 }
 161
 162 static int unhex(unsigned char *c2)
 163 {
 164         int x, c;
 165         x = *c2++ & 0x4f;
 166         if (x & 0x40)
 167                 x -= 55;
 168         c = x << 4;
 169         x = *c2 & 0x4f;
 170         if (x & 0x40)
 171                 x -= 55;
 172         c |= x;
 173         return c;
 174 }
 175
 176 static int readline(MDB_val *out, MDB_val *buf)
 177 {
 178         unsigned char *c1, *c2, *end;
 179         size_t len, l2;
 180         int c;
 181
 182         if (!(mode & NOHDR)) {
 183                 c = fgetc(stdin);
 184                 if (c == EOF) {
 185                         Eof = 1;
 186                         return EOF;
 187                 }
 188                 if (c != ' ') {
 189                         lineno++;
 190                         if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
 191 badend:
 192                                 Eof = 1;
 193                                 badend();
 194                                 return EOF;
 195                         }
 196                         if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END")))
 197                                 return EOF;
 198                         goto badend;
 199                 }
 200         }
 201         if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
 202                 Eof = 1;
 203                 return EOF;
 204         }
 205         lineno++;
 206
 207         c1 = buf->mv_data;
 208         len = strlen((char *)c1);
 209         l2 = len;
 210
 211         /* Is buffer too short? */
 212         while (c1[len-1] != '\n') {
 213                 buf->mv_data = realloc(buf->mv_data, buf->mv_size*2);
 214                 if (!buf->mv_data) {
 215                         Eof = 1;
 216                         fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n",
 217                                 prog, lineno);
 218                         return EOF;
 219                 }
 220                 c1 = buf->mv_data;
 221                 c1 += l2;
 222                 if (fgets((char *)c1, buf->mv_size+1, stdin) == NULL) {
 223                         Eof = 1;
 224                         badend();
 225                         return EOF;
 226                 }
 227                 buf->mv_size *= 2;
 228                 len = strlen((char *)c1);
 229                 l2 += len;
 230         }
 231         c1 = c2 = buf->mv_data;
 232         len = l2;
 233         c1[--len] = '\0';
 234         end = c1 + len;
 235
 236         if (mode & PRINT) {
 237                 while (c2 < end) {
 238                         if (*c2 == '\\') {
 239                                 if (c2[1] == '\\') {
 240                                         c1++; c2 += 2;
 241                                 } else {
 242                                         if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
 243                                                 Eof = 1;
 244                                                 badend();
 245                                                 return EOF;
 246                                         }
 247                                         *c1++ = unhex(++c2);
 248                                         c2 += 2;
 249                                 }
 250                         } else {
 251                                 /* copies are redundant when no escapes were used */
 252                                 *c1++ = *c2++;
 253                         }
 254                 }
 255         } else {
 256                 /* odd length not allowed */
 257                 if (len & 1) {
 258                         Eof = 1;
 259                         badend();
 260                         return EOF;
 261                 }
 262                 while (c2 < end) {
 263                         if (!isxdigit(*c2) || !isxdigit(c2[1])) {
 264                                 Eof = 1;
 265                                 badend();
 266                                 return EOF;
 267                         }
 268                         *c1++ = unhex(c2);
 269                         c2 += 2;
 270                 }
 271         }
 272         c2 = out->mv_data = buf->mv_data;
 273         out->mv_size = c1 - c2;
 274
 275         return 0;
 276 }
 277
 278 static void usage(void)
 279 {
 280         fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
 281         exit(EXIT_FAILURE);
 282 }
 283
 284 int main(int argc, char *argv[])
 285 {
 286         int i, rc;
 287         MDB_env *env;
 288         MDB_txn *txn;
 289         MDB_cursor *mc;
 290         MDB_dbi dbi;
 291         char *envname;
 292         int envflags = 0, putflags = 0;
 293         int dohdr = 0;
 294
 295         prog = argv[0];
 296
 297         if (argc < 2) {
 298                 usage();
 299         }
 300
 301         /* -f: load file instead of stdin
 302          * -n: use NOSUBDIR flag on env_open
 303          * -s: load into named subDB
 304          * -N: use NOOVERWRITE on puts
 305          * -T: read plaintext
 306          * -V: print version and exit
 307          */
 308         while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
 309                 switch(i) {
 310                 case 'V':
 311                         printf("%s\n", MDB_VERSION_STRING);
 312                         exit(0);
 313                         break;
 314                 case 'f':
 315                         if (freopen(optarg, "r", stdin) == NULL) {
 316                                 fprintf(stderr, "%s: %s: reopen: %s\n",
 317                                         prog, optarg, strerror(errno));
 318                                 exit(EXIT_FAILURE);
 319                         }
 320                         break;
 321                 case 'n':
 322                         envflags |= MDB_NOSUBDIR;
 323                         break;
 324                 case 's':
 325                         subname = strdup(optarg);
 326                         break;
 327                 case 'N':
 328                         putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
 329                         break;
 330                 case 'T':
 331                         mode |= NOHDR | PRINT;
 332                         break;
 333                 default:
 334                         usage();
 335                 }
 336         }
 337
 338         if (optind != argc - 1)
 339                 usage();
 340
 341         dbuf.mv_size = 4096;
 342         dbuf.mv_data = malloc(dbuf.mv_size);
 343
 344         if (!(mode & NOHDR))
 345                 readhdr();
 346
 347         envname = argv[optind];
 348         rc = mdb_env_create(&env);
 349         if (rc) {
 350                 fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
 351                 return EXIT_FAILURE;
 352         }
 353
 354         mdb_env_set_maxdbs(env, 2);
 355
 356         if (info.me_maxreaders)
 357                 mdb_env_set_maxreaders(env, info.me_maxreaders);
 358
 359         if (info.me_mapsize)
 360                 mdb_env_set_mapsize(env, info.me_mapsize);
 361
 362         if (info.me_mapaddr)
 363                 envflags |= MDB_FIXEDMAP;
 364
 365         rc = mdb_env_open(env, envname, envflags, 0664);
 366         if (rc) {
 367                 fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
 368                 goto env_close;
 369         }
 370
 371         kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
 372         kbuf.mv_data = malloc(kbuf.mv_size);
 373
 374         while(!Eof) {
 375                 MDB_val key, data;
 376                 int batch = 0;
 377                 flags = 0;
 378
 379                 if (!dohdr) {
 380                         dohdr = 1;
 381                 } else if (!(mode & NOHDR))
 382                         readhdr();
 383
 384                 rc = mdb_txn_begin(env, NULL, 0, &txn);
 385                 if (rc) {
 386                         fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
 387                         goto env_close;
 388                 }
 389
 390                 rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi);
 391                 if (rc) {
 392                         fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
 393                         goto txn_abort;
 394                 }
 395
 396                 rc = mdb_cursor_open(txn, dbi, &mc);
 397                 if (rc) {
 398                         fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
 399                         goto txn_abort;
 400                 }
 401
 402                 while(1) {
 403                         rc = readline(&key, &kbuf);
 404                         if (rc)  /* rc == EOF */
 405                                 break;
 406
 407                         rc = readline(&data, &dbuf);
 408                         if (rc) {
 409                                 fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno);
 410                                 goto txn_abort;
 411                         }
 412
 413                         rc = mdb_cursor_put(mc, &key, &data, putflags);
 414                         if (rc == MDB_KEYEXIST && putflags)
 415                                 continue;
 416                         if (rc) {
 417                                 fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
 418                                 goto txn_abort;
 419                         }
 420                         batch++;
 421                         if (batch == 100) {
 422                                 rc = mdb_txn_commit(txn);
 423                                 if (rc) {
 424                                         fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
 425                                                 prog, lineno, mdb_strerror(rc));
 426                                         goto env_close;
 427                                 }
 428                                 rc = mdb_txn_begin(env, NULL, 0, &txn);
 429                                 if (rc) {
 430                                         fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
 431                                         goto env_close;
 432                                 }
 433                                 rc = mdb_cursor_open(txn, dbi, &mc);
 434                                 if (rc) {
 435                                         fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
 436                                         goto txn_abort;
 437                                 }
 438                                 batch = 0;
 439                         }
 440                 }
 441                 rc = mdb_txn_commit(txn);
 442                 txn = NULL;
 443                 if (rc) {
 444                         fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
 445                                 prog, lineno, mdb_strerror(rc));
 446                         goto env_close;
 447                 }
 448                 mdb_dbi_close(env, dbi);
 449         }
 450
 451 txn_abort:
 452         mdb_txn_abort(txn);
 453 env_close:
 454         mdb_env_close(env);
 455
 456         return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 457 }