git.sur5r.net Git - openldap/blob - libraries/liblmdb/mdb_load.c

   1 /* mdb_load.c - memory-mapped database load tool */
   2 /*
   3  * Copyright 2011-2016 Howard Chu, Symas Corp.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted only as authorized by the OpenLDAP
   8  * Public License.
   9  *
  10  * A copy of this license is available in the file LICENSE in the
  11  * top-level directory of the distribution or, alternatively, at
  12  * <http://www.OpenLDAP.org/license.html>.
  13  */
  14 #include <stdio.h>
  15 #include <stdlib.h>
  16 #include <errno.h>
  17 #include <string.h>
  18 #include <ctype.h>
  19 #include <unistd.h>
  20 #include "lmdb.h"
  21
  22 #define PRINT   1
  23 #define NOHDR   2
  24 static int mode;
  25
  26 static char *subname = NULL;
  27
  28 static size_t lineno;
  29 static int version;
  30
  31 static int flags;
  32
  33 static char *prog;
  34
  35 static int Eof;
  36
  37 static MDB_envinfo info;
  38
  39 static MDB_val kbuf, dbuf;
  40
  41 #ifdef _WIN32
  42 #define Z       "I"
  43 #else
  44 #define Z       "z"
  45 #endif
  46 #ifdef MDB_VL32
  47 #ifdef _WIN32
  48 #define Y       "I64"
  49 #else
  50 #define Y       "ll"
  51 #endif
  52 #else
  53 #define Y       Z
  54 #endif
  55
  56 #define STRLENOF(s)     (sizeof(s)-1)
  57
  58 typedef struct flagbit {
  59         int bit;
  60         char *name;
  61         int len;
  62 } flagbit;
  63
  64 #define S(s)    s, STRLENOF(s)
  65
  66 flagbit dbflags[] = {
  67         { MDB_REVERSEKEY, S("reversekey") },
  68         { MDB_DUPSORT, S("dupsort") },
  69         { MDB_INTEGERKEY, S("integerkey") },
  70         { MDB_DUPFIXED, S("dupfixed") },
  71         { MDB_INTEGERDUP, S("integerdup") },
  72         { MDB_REVERSEDUP, S("reversedup") },
  73         { 0, NULL, 0 }
  74 };
  75
  76 static void readhdr(void)
  77 {
  78         char *ptr;
  79
  80         while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) {
  81                 lineno++;
  82                 if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
  83                         version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION="));
  84                         if (version > 3) {
  85                                 fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n",
  86                                         prog, lineno, version);
  87                                 exit(EXIT_FAILURE);
  88                         }
  89                 } else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) {
  90                         break;
  91                 } else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) {
  92                         if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print")))
  93                                 mode |= PRINT;
  94                         else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) {
  95                                 fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n",
  96                                         prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT="));
  97                                 exit(EXIT_FAILURE);
  98                         }
  99                 } else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) {
 100                         ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
 101                         if (ptr) *ptr = '\0';
 102                         if (subname) free(subname);
 103                         subname = strdup((char *)dbuf.mv_data+STRLENOF("database="));
 104                 } else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) {
 105                         if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree")))  {
 106                                 fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n",
 107                                         prog, lineno, (char *)dbuf.mv_data+STRLENOF("type="));
 108                                 exit(EXIT_FAILURE);
 109                         }
 110                 } else if (!strncmp(dbuf.mv_data, "mapaddr=", STRLENOF("mapaddr="))) {
 111                         int i;
 112                         ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
 113                         if (ptr) *ptr = '\0';
 114                         i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr);
 115                         if (i != 1) {
 116                                 fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n",
 117                                         prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr="));
 118                                 exit(EXIT_FAILURE);
 119                         }
 120                 } else if (!strncmp(dbuf.mv_data, "mapsize=", STRLENOF("mapsize="))) {
 121                         int i;
 122                         ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
 123                         if (ptr) *ptr = '\0';
 124                         i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Y "u", &info.me_mapsize);
 125                         if (i != 1) {
 126                                 fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n",
 127                                         prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
 128                                 exit(EXIT_FAILURE);
 129                         }
 130                 } else if (!strncmp(dbuf.mv_data, "maxreaders=", STRLENOF("maxreaders="))) {
 131                         int i;
 132                         ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
 133                         if (ptr) *ptr = '\0';
 134                         i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders);
 135                         if (i != 1) {
 136                                 fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n",
 137                                         prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders="));
 138                                 exit(EXIT_FAILURE);
 139                         }
 140                 } else {
 141                         int i;
 142                         for (i=0; dbflags[i].bit; i++) {
 143                                 if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) &&
 144                                         ((char *)dbuf.mv_data)[dbflags[i].len] == '=') {
 145                                         flags |= dbflags[i].bit;
 146                                         break;
 147                                 }
 148                         }
 149                         if (!dbflags[i].bit) {
 150                                 ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size);
 151                                 if (!ptr) {
 152                                         fprintf(stderr, "%s: line %" Z "d: unexpected format\n",
 153                                                 prog, lineno);
 154                                         exit(EXIT_FAILURE);
 155                                 } else {
 156                                         *ptr = '\0';
 157                                         fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n",
 158                                                 prog, lineno, (char *)dbuf.mv_data);
 159                                 }
 160                         }
 161                 }
 162         }
 163 }
 164
 165 static void badend(void)
 166 {
 167         fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n",
 168                 prog, lineno);
 169 }
 170
 171 static int unhex(unsigned char *c2)
 172 {
 173         int x, c;
 174         x = *c2++ & 0x4f;
 175         if (x & 0x40)
 176                 x -= 55;
 177         c = x << 4;
 178         x = *c2 & 0x4f;
 179         if (x & 0x40)
 180                 x -= 55;
 181         c |= x;
 182         return c;
 183 }
 184
 185 static int readline(MDB_val *out, MDB_val *buf)
 186 {
 187         unsigned char *c1, *c2, *end;
 188         size_t len, l2;
 189         int c;
 190
 191         if (!(mode & NOHDR)) {
 192                 c = fgetc(stdin);
 193                 if (c == EOF) {
 194                         Eof = 1;
 195                         return EOF;
 196                 }
 197                 if (c != ' ') {
 198                         lineno++;
 199                         if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
 200 badend:
 201                                 Eof = 1;
 202                                 badend();
 203                                 return EOF;
 204                         }
 205                         if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END")))
 206                                 return EOF;
 207                         goto badend;
 208                 }
 209         }
 210         if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
 211                 Eof = 1;
 212                 return EOF;
 213         }
 214         lineno++;
 215
 216         c1 = buf->mv_data;
 217         len = strlen((char *)c1);
 218         l2 = len;
 219
 220         /* Is buffer too short? */
 221         while (c1[len-1] != '\n') {
 222                 buf->mv_data = realloc(buf->mv_data, buf->mv_size*2);
 223                 if (!buf->mv_data) {
 224                         Eof = 1;
 225                         fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n",
 226                                 prog, lineno);
 227                         return EOF;
 228                 }
 229                 c1 = buf->mv_data;
 230                 c1 += l2;
 231                 if (fgets((char *)c1, buf->mv_size+1, stdin) == NULL) {
 232                         Eof = 1;
 233                         badend();
 234                         return EOF;
 235                 }
 236                 buf->mv_size *= 2;
 237                 len = strlen((char *)c1);
 238                 l2 += len;
 239         }
 240         c1 = c2 = buf->mv_data;
 241         len = l2;
 242         c1[--len] = '\0';
 243         end = c1 + len;
 244
 245         if (mode & PRINT) {
 246                 while (c2 < end) {
 247                         if (*c2 == '\\') {
 248                                 if (c2[1] == '\\') {
 249                                         c1++; c2 += 2;
 250                                 } else {
 251                                         if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
 252                                                 Eof = 1;
 253                                                 badend();
 254                                                 return EOF;
 255                                         }
 256                                         *c1++ = unhex(++c2);
 257                                         c2 += 2;
 258                                 }
 259                         } else {
 260                                 c1++; c2++;
 261                         }
 262                 }
 263         } else {
 264                 /* odd length not allowed */
 265                 if (len & 1) {
 266                         Eof = 1;
 267                         badend();
 268                         return EOF;
 269                 }
 270                 while (c2 < end) {
 271                         if (!isxdigit(*c2) || !isxdigit(c2[1])) {
 272                                 Eof = 1;
 273                                 badend();
 274                                 return EOF;
 275                         }
 276                         *c1++ = unhex(c2);
 277                         c2 += 2;
 278                 }
 279         }
 280         c2 = out->mv_data = buf->mv_data;
 281         out->mv_size = c1 - c2;
 282
 283         return 0;
 284 }
 285
 286 static void usage(void)
 287 {
 288         fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
 289         exit(EXIT_FAILURE);
 290 }
 291
 292 int main(int argc, char *argv[])
 293 {
 294         int i, rc;
 295         MDB_env *env;
 296         MDB_txn *txn;
 297         MDB_cursor *mc;
 298         MDB_dbi dbi;
 299         char *envname;
 300         int envflags = 0, putflags = 0;
 301         int dohdr = 0;
 302
 303         prog = argv[0];
 304
 305         if (argc < 2) {
 306                 usage();
 307         }
 308
 309         /* -f: load file instead of stdin
 310          * -n: use NOSUBDIR flag on env_open
 311          * -s: load into named subDB
 312          * -N: use NOOVERWRITE on puts
 313          * -T: read plaintext
 314          * -V: print version and exit
 315          */
 316         while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
 317                 switch(i) {
 318                 case 'V':
 319                         printf("%s\n", MDB_VERSION_STRING);
 320                         exit(0);
 321                         break;
 322                 case 'f':
 323                         if (freopen(optarg, "r", stdin) == NULL) {
 324                                 fprintf(stderr, "%s: %s: reopen: %s\n",
 325                                         prog, optarg, strerror(errno));
 326                                 exit(EXIT_FAILURE);
 327                         }
 328                         break;
 329                 case 'n':
 330                         envflags |= MDB_NOSUBDIR;
 331                         break;
 332                 case 's':
 333                         subname = strdup(optarg);
 334                         break;
 335                 case 'N':
 336                         putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
 337                         break;
 338                 case 'T':
 339                         mode |= NOHDR | PRINT;
 340                         break;
 341                 default:
 342                         usage();
 343                 }
 344         }
 345
 346         if (optind != argc - 1)
 347                 usage();
 348
 349         dbuf.mv_size = 4096;
 350         dbuf.mv_data = malloc(dbuf.mv_size);
 351
 352         if (!(mode & NOHDR))
 353                 readhdr();
 354
 355         envname = argv[optind];
 356         rc = mdb_env_create(&env);
 357         if (rc) {
 358                 fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
 359                 return EXIT_FAILURE;
 360         }
 361
 362         mdb_env_set_maxdbs(env, 2);
 363
 364         if (info.me_maxreaders)
 365                 mdb_env_set_maxreaders(env, info.me_maxreaders);
 366
 367         if (info.me_mapsize)
 368                 mdb_env_set_mapsize(env, info.me_mapsize);
 369
 370         if (info.me_mapaddr)
 371                 envflags |= MDB_FIXEDMAP;
 372
 373         rc = mdb_env_open(env, envname, envflags, 0664);
 374         if (rc) {
 375                 fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
 376                 goto env_close;
 377         }
 378
 379         kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
 380         kbuf.mv_data = malloc(kbuf.mv_size);
 381
 382         while(!Eof) {
 383                 MDB_val key, data;
 384                 int batch = 0;
 385                 flags = 0;
 386
 387                 if (!dohdr) {
 388                         dohdr = 1;
 389                 } else if (!(mode & NOHDR))
 390                         readhdr();
 391
 392                 rc = mdb_txn_begin(env, NULL, 0, &txn);
 393                 if (rc) {
 394                         fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
 395                         goto env_close;
 396                 }
 397
 398                 rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi);
 399                 if (rc) {
 400                         fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
 401                         goto txn_abort;
 402                 }
 403
 404                 rc = mdb_cursor_open(txn, dbi, &mc);
 405                 if (rc) {
 406                         fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
 407                         goto txn_abort;
 408                 }
 409
 410                 while(1) {
 411                         rc = readline(&key, &kbuf);
 412                         if (rc)  /* rc == EOF */
 413                                 break;
 414
 415                         rc = readline(&data, &dbuf);
 416                         if (rc) {
 417                                 fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno);
 418                                 goto txn_abort;
 419                         }
 420
 421                         rc = mdb_cursor_put(mc, &key, &data, putflags);
 422                         if (rc == MDB_KEYEXIST && putflags)
 423                                 continue;
 424                         if (rc) {
 425                                 fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
 426                                 goto txn_abort;
 427                         }
 428                         batch++;
 429                         if (batch == 100) {
 430                                 rc = mdb_txn_commit(txn);
 431                                 if (rc) {
 432                                         fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
 433                                                 prog, lineno, mdb_strerror(rc));
 434                                         goto env_close;
 435                                 }
 436                                 rc = mdb_txn_begin(env, NULL, 0, &txn);
 437                                 if (rc) {
 438                                         fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
 439                                         goto env_close;
 440                                 }
 441                                 rc = mdb_cursor_open(txn, dbi, &mc);
 442                                 if (rc) {
 443                                         fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
 444                                         goto txn_abort;
 445                                 }
 446                                 batch = 0;
 447                         }
 448                 }
 449                 rc = mdb_txn_commit(txn);
 450                 txn = NULL;
 451                 if (rc) {
 452                         fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
 453                                 prog, lineno, mdb_strerror(rc));
 454                         goto env_close;
 455                 }
 456                 mdb_dbi_close(env, dbi);
 457         }
 458
 459 txn_abort:
 460         mdb_txn_abort(txn);
 461 env_close:
 462         mdb_env_close(env);
 463
 464         return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 465 }