X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=bacula%2Fsrc%2Ffiled%2Faccurate.c;h=e404d18f5e03c9190e3664d77db58e7caf1e2952;hb=f5571f4e4916261503020df01b6f9d30d17ae194;hp=5c78f8eb8accd392c4aea7f37c0fbefb52d1549e;hpb=f5111f8ddd61d85a7f115900fd7d7d37a32f59eb;p=bacula%2Fbacula

diff --git a/bacula/src/filed/accurate.c b/bacula/src/filed/accurate.c
index 5c78f8eb8a..e404d18f5e 100644
--- a/bacula/src/filed/accurate.c
+++ b/bacula/src/filed/accurate.c
@@ -33,8 +33,10 @@
 #include "bacula.h"
 #include "filed.h"
 
+static int dbglvl=200;
+
 typedef struct PrivateCurFile {
-#ifndef USE_TCHDB
+#ifndef USE_TCADB
    hlink link;
 #endif
    char *fname;			/* not stored with tchdb mode */
@@ -43,117 +45,261 @@ typedef struct PrivateCurFile {
    bool seen;
 } CurFile;
 
-bool accurate_add_file(JCR *jcr, char *fname, char *lstat)
-{
-   CurFile elt;
-   struct stat statp;
-   int LinkFIc;
-   decode_stat(lstat, &statp, &LinkFIc); /* decode catalog stat */
-   elt.ctime = statp.st_ctime;
-   elt.mtime = statp.st_mtime;
-   elt.seen = 0;
-
-#ifdef USE_TCHDB
-   if (!tchdbputasync(jcr->file_list,
-		      fname, strlen(fname)+1,
-		      &elt, sizeof(CurFile)))
-   {
-      Dmsg1(2, "Can't add <%s> to file_list\n", fname);
-      /* TODO: check error */
-   }
-#else  /* HTABLE */
-   CurFile *item;
-   /* we store CurFile, fname and ctime/mtime in the same chunk */
-   item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+strlen(fname)+1);
-   memcpy(item, &elt, sizeof(CurFile));
-   item->fname  = (char *)item+sizeof(CurFile);
-   strcpy(item->fname, fname);
-   jcr->file_list->insert(item->fname, item); 
-#endif
-
-   Dmsg2(2, "add fname=<%s> lstat=%s\n", fname, lstat);
-   return true;
-}
+#ifdef USE_TCADB
+static void realfree(void *p);	/* used by tokyo code */
 
-bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
+/*
+ * Update hash element seen=1
+ */
+static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
 {
    bool ret=true;
-   CurFile deb;
 
-#ifdef USE_TCHDB
    elt->seen = 1;
-   if (!tchdbputasync(jcr->file_list, 
-		      elt->fname, strlen(elt->fname)+1, 
-		      elt, sizeof(CurFile)))
-   {
-      Dmsg1(2, "can't update <%s>\n", elt->fname);
-      ret = false;		/* TODO: add error message */
+   if (!tcadbput(jcr->file_list, 
+		 elt->fname, strlen(elt->fname)+1, 
+		 elt, sizeof(CurFile)))
+   { /* TODO: disabling accurate mode ?  */
+      Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk\n"));
+      ret = false;
    }
 
-#else  /* HTABLE */
-   CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
-   temp->seen = 1;
-#endif    
    return ret;
 }
 
-bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
+static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
 {
    bool found=false;
    ret->seen = 0;
+   int size;
+   CurFile *elt;
 
-#ifdef USE_TCHDB
-   if (tchdbget3(jcr->file_list, 
-		 fname, strlen(fname)+1, 
-		 ret, sizeof(CurFile)) != -1)
+   elt = (CurFile*)tcadbget(jcr->file_list, 
+			    fname, strlen(fname)+1, &size);
+   if (elt)
    {
+      /* TODO: don't malloc/free results */
       found = true;
-      ret->fname = fname;
+      elt->fname = fname;
+      memcpy(ret, elt, sizeof(CurFile));
+      realfree(elt);
+//    Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
    }
+   return found;
+}
+
+/* Create tokyo dbm hash file 
+ * If something goes wrong, we cancel accurate mode.
+ */
+static bool accurate_init(JCR *jcr, int nbfile)
+{
+   jcr->file_list = tcadbnew();
+//
+//   tchdbsetcache(jcr->file_list, 300000);
+//   tchdbtune(jcr->file_list,
+//	     nbfile,		/* nb bucket 0.5n to 4n */
+//	     6,			/* size of element 2^x */
+//	     16,
+//	     0);		/* options like compression */
+//
+   jcr->hash_name  = get_pool_memory(PM_MESSAGE);
+   POOLMEM *temp = get_pool_memory(PM_MESSAGE);
+
+   if (nbfile > 500000) {
+      make_unique_filename(&jcr->hash_name, jcr->JobId, "accurate");
+      pm_strcat(jcr->hash_name, ".tcb");
+      Mmsg(temp, "%s#bnum=%i#mode=e#opts=l",
+	   jcr->hash_name, nbfile*4); 
+      Dmsg1(dbglvl, "Doing accurate hash on disk %s\n", jcr->hash_name);
+   } else {
+      Dmsg0(dbglvl, "Doing accurate hash on memory\n");
+      pm_strcpy(jcr->hash_name, "*");
+      pm_strcpy(temp, "*");
+   }
+   
+   if(!tcadbopen(jcr->file_list, jcr->hash_name)){
+      Jmsg(jcr, M_ERROR, 1, _("Can't open accurate hash disk\n"));
+      Jmsg(jcr, M_INFO, 1, _("Disabling accurate mode\n"));
+      tcadbdel(jcr->file_list);
+      jcr->file_list = NULL;
+      jcr->accurate = false;
+   }
+   free_pool_memory(temp);
+   return jcr->file_list != NULL;
+}
+
+/* This function is called at the end of backup
+ * We walk over all hash disk element, and we check
+ * for elt.seen.
+ */
+bool accurate_send_deleted_list(JCR *jcr)
+{
+   char *key;
+   CurFile *elt;
+   int size;
+   FF_PKT *ff_pkt;
+   int stream = STREAM_UNIX_ATTRIBUTES;
+
+   if (!jcr->accurate || jcr->JobLevel == L_FULL) {
+      goto bail_out;
+   }
+
+   if (jcr->file_list == NULL) {
+      goto bail_out;
+   }
+
+   ff_pkt = init_find_files();
+   ff_pkt->type = FT_DELETED;
+
+   /* traverse records */
+   tcadbiterinit(jcr->file_list);
+   while((key = tcadbiternext2(jcr->file_list)) != NULL) {
+      elt = (CurFile *) tcadbget(jcr->file_list, 
+				 key, strlen(key)+1, &size);
+      if (elt)
+      {
+	 if (!elt->seen) {	/* already seen */
+	    ff_pkt->fname = key;
+	    ff_pkt->statp.st_mtime = elt->mtime;
+	    ff_pkt->statp.st_ctime = elt->ctime;
+	    encode_and_send_attributes(jcr, ff_pkt, stream);
+	 }
+	 realfree(elt);
+      }
+      realfree(key);		/* tokyo cabinet have to use real free() */
+   }
+
+   term_find_files(ff_pkt);
+bail_out:
+   /* TODO: clean htable when this function is not reached ? */
+   if (jcr->file_list) {
+      if(!tcadbclose(jcr->file_list)){
+	 Jmsg(jcr, M_ERROR, 1, _("Can't close accurate hash disk\n"));
+      }
+
+      /* delete the object */
+      tcadbdel(jcr->file_list);
+      if (!bstrcmp(jcr->hash_name, "*")) {
+	 unlink(jcr->hash_name);
+      }
+
+      free_pool_memory(jcr->hash_name);
+      jcr->hash_name = NULL;
+      jcr->file_list = NULL;
+   }
+   return true;
+}
+
+#else  /* HTABLE mode */
+
+static bool accurate_mark_file_as_seen(JCR *jcr, CurFile *elt)
+{
+   CurFile *temp = (CurFile *)jcr->file_list->lookup(elt->fname);
+   temp->seen = 1;		/* records are in memory */
+   return true;
+}
+
+static bool accurate_lookup(JCR *jcr, char *fname, CurFile *ret)
+{
+   bool found=false;
+   ret->seen = 0;
 
-#else  /* HTABLE */
    CurFile *temp = (CurFile *)jcr->file_list->lookup(fname);
    if (temp) {
       memcpy(ret, temp, sizeof(CurFile));
       found=true;
-   }
-#endif
-
-   if (found) {
-      Dmsg1(2, "lookup <%s> ok\n", fname);
+//    Dmsg1(dbglvl, "lookup <%s> ok\n", fname);
    }
 
    return found;
 }
 
-bool accurate_init(JCR *jcr, int nbfile)
+static bool accurate_init(JCR *jcr, int nbfile)
 {
-#ifdef USE_TCHDB
-   jcr->file_list = tchdbnew();
-   tchdbsetcache(jcr->file_list, 300000);
-   tchdbtune(jcr->file_list,
-	     nbfile,		/* nb bucket 0.5n to 4n */
-	     7,			/* size of element 2^x */
-	     16,
-	     0);		/* options like compression */
-   /* TODO: make accurate file unique */
-   POOL_MEM buf;
-   Mmsg(buf, "/tmp/casket.hdb.%i", jcr->JobId);
-   if(!tchdbopen(jcr->file_list, buf.c_str(), HDBOWRITER | HDBOCREAT)){
-      /* TODO: handle error creation */
-      //ecode = tchdbecode(hdb);
-      //fprintf(stderr, "open error: %s\n", tchdberrmsg(ecode));
-   }
-
-#else  /* HTABLE */
    CurFile *elt=NULL;
    jcr->file_list = (htable *)malloc(sizeof(htable));
    jcr->file_list->init(elt, &elt->link, nbfile);
-#endif
+   return true;
+}
+
+/* This function is called at the end of backup
+ * We walk over all hash disk element, and we check
+ * for elt.seen.
+ */
+bool accurate_send_deleted_list(JCR *jcr)
+{
+   CurFile *elt;
+   FF_PKT *ff_pkt;
+   int stream = STREAM_UNIX_ATTRIBUTES;
 
+   if (!jcr->accurate || jcr->JobLevel == L_FULL) {
+      goto bail_out;
+   }
+
+   if (jcr->file_list == NULL) {
+      goto bail_out;
+   }
+
+   ff_pkt = init_find_files();
+   ff_pkt->type = FT_DELETED;
+
+   foreach_htable (elt, jcr->file_list) {
+      if (!elt->seen) { /* already seen */
+         Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
+         ff_pkt->fname = elt->fname;
+         ff_pkt->statp.st_mtime = elt->mtime;
+         ff_pkt->statp.st_ctime = elt->ctime;
+         encode_and_send_attributes(jcr, ff_pkt, stream);
+      }
+//      free(elt->fname);
+   }
+
+   term_find_files(ff_pkt);
+bail_out:
+   /* TODO: clean htable when this function is not reached ? */
+   if (jcr->file_list) {
+      jcr->file_list->destroy();
+      free(jcr->file_list);
+      jcr->file_list = NULL;
+   }
    return true;
 }
 
+#endif /* common code */
+
+static bool accurate_add_file(JCR *jcr, char *fname, char *lstat)
+{
+   bool ret = true;
+   CurFile elt;
+   struct stat statp;
+   int LinkFIc;
+   decode_stat(lstat, &statp, &LinkFIc); /* decode catalog stat */
+   elt.ctime = statp.st_ctime;
+   elt.mtime = statp.st_mtime;
+   elt.seen = 0;
+
+#ifdef USE_TCADB
+   if (!tcadbput(jcr->file_list,
+		 fname, strlen(fname)+1,
+		 &elt, sizeof(CurFile)))
+   {
+      Jmsg(jcr, M_ERROR, 1, _("Can't update accurate hash disk ERR=%s\n"));
+      ret = false;
+   }
+#else  /* HTABLE */
+   CurFile *item;
+   /* we store CurFile, fname and ctime/mtime in the same chunk */
+   item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+strlen(fname)+1);
+   memcpy(item, &elt, sizeof(CurFile));
+   item->fname  = (char *)item+sizeof(CurFile);
+   strcpy(item->fname, fname);
+   jcr->file_list->insert(item->fname, item); 
+#endif
+
+// Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat);
+   return ret;
+}
+
 /*
  * This function is called for each file seen in fileset.
  * We check in file_list hash if fname have been backuped
@@ -179,13 +325,13 @@ bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
    } 
 
    if (!accurate_lookup(jcr, fname, &elt)) {
-      Dmsg1(2, "accurate %s (not found)\n", fname);
+      Dmsg1(dbglvl, "accurate %s (not found)\n", fname);
       stat = true;
       goto bail_out;
    }
 
    if (elt.seen) { /* file has been seen ? */
-      Dmsg1(2, "accurate %s (already seen)\n", fname);
+      Dmsg1(dbglvl, "accurate %s (already seen)\n", fname);
       goto bail_out;
    }
 
@@ -197,13 +343,8 @@ bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
      stat = true;
    }
 
-   if (stat) {
-      Dmsg4(1, "%i = %i\t%i = %i\n", elt.mtime, ff_pkt->statp.st_mtime,
-	    elt.ctime, ff_pkt->statp.st_ctime);
-   }
-
    accurate_mark_file_as_seen(jcr, &elt);
-   Dmsg2(2, "accurate %s = %i\n", fname, stat);
+   Dmsg2(dbglvl, "accurate %s = %i\n", fname, stat);
 
 bail_out:
    unstrip_path(ff_pkt);
@@ -227,7 +368,6 @@ int accurate_cmd(JCR *jcr)
       dir->fsend(_("2991 Bad accurate command\n"));
       return false;
    }
-   Dmsg2(2, "nb=%d msg=%s\n", nb, dir->msg);
 
    accurate_init(jcr, nb);
 
@@ -237,7 +377,6 @@ int accurate_cmd(JCR *jcr)
     */
    /* get current files */
    while (dir->recv() >= 0) {
-      Dmsg1(2, "accurate_cmd fname=%s\n", dir->msg);
       len = strlen(dir->msg) + 1;
       if (len < dir->msglen) {
 	 accurate_add_file(jcr, dir->msg, dir->msg + len);
@@ -248,7 +387,7 @@ int accurate_cmd(JCR *jcr)
    extern void *start_heap;
 
    char b1[50], b2[50], b3[50], b4[50], b5[50];
-   Dmsg5(1," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
+   Dmsg5(dbglvl," Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n",
          edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1),
          edit_uint64_with_commas(sm_bytes, b2),
          edit_uint64_with_commas(sm_max_bytes, b3),
@@ -260,80 +399,16 @@ int accurate_cmd(JCR *jcr)
    return true;
 }
 
-bool accurate_send_deleted_list(JCR *jcr)
-{
-   CurFile *elt;
-   FF_PKT *ff_pkt;
-   int stream = STREAM_UNIX_ATTRIBUTES;
-
-   if (!jcr->accurate || jcr->JobLevel == L_FULL) {
-      goto bail_out;
-   }
-
-   if (jcr->file_list == NULL) {
-      goto bail_out;
-   }
-
-   ff_pkt = init_find_files();
-   ff_pkt->type = FT_DELETED;
-
-#ifdef USE_TCHDB
-   char *key;
-   CurFile item;
-   elt = &item;
-  /* traverse records */
-   tchdbiterinit(jcr->file_list);
-   while((key = tchdbiternext2(jcr->file_list)) != NULL) {
-      if (tchdbget3(jcr->file_list, 
-		    key, strlen(key)+1, 
-		    elt, sizeof(CurFile)) != -1)
-      {
-	 if (!elt->seen) {
-	    ff_pkt->fname = key;
-	    ff_pkt->statp.st_mtime = elt->mtime;
-	    ff_pkt->statp.st_ctime = elt->ctime;
-	    encode_and_send_attributes(jcr, ff_pkt, stream);
-	    Dmsg1(2, "deleted <%s>\n", key);
-	 }
- //      free(key);
-
-      } else {			/* TODO: add error message */
-	 Dmsg1(2, "No value for <%s> key\n", key);
-      }
-   }
-#else
-   foreach_htable (elt, jcr->file_list) {
-      if (!elt->seen) { /* already seen */
-         Dmsg2(1, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
-         ff_pkt->fname = elt->fname;
-         ff_pkt->statp.st_mtime = elt->mtime;
-         ff_pkt->statp.st_ctime = elt->ctime;
-         encode_and_send_attributes(jcr, ff_pkt, stream);
-      }
-//      free(elt->fname);
-   }
-#endif
+#ifdef USE_TCADB
 
-   term_find_files(ff_pkt);
-bail_out:
-   /* TODO: clean htable when this function is not reached ? */
-   if (jcr->file_list) {
-#ifdef USE_TCHDB
-      if(!tchdbclose(jcr->file_list)){
-//	 ecode = tchdbecode(hdb);
-//	 fprintf(stderr, "close error: %s\n", tchdberrmsg(ecode));
-      }
+/*
+ * Tokyo Cabinet library doesn't use smartalloc by default
+ * results need to be released with real free()
+ */
+#undef free
+void realfree(void *p)
+{
+   free(p);
+}
 
-      /* delete the object */
-      tchdbdel(jcr->file_list);
-      POOL_MEM buf;
-      Mmsg(buf, "/tmp/casket.hdb.%i", jcr->JobId);
-//      unlink("/tmp/casket.hdb");
-#else
-      jcr->file_list->destroy();
-      free(jcr->file_list);
 #endif
-      jcr->file_list = NULL;
-   }
-   return true;
-}