From 823baf4ff081c1932a362322323f3fb63787a5fc Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Tue, 14 Jan 2003 10:04:08 +0000 Subject: [PATCH] Enhance dbcheck -- tweak Paths git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@286 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/ChangeLog | 3 +- bacula/ReleaseNotes | 36 +----------- bacula/kernstodo | 73 ++----------------------- bacula/src/cats/sql.c | 9 ++- bacula/src/cats/sql_create.c | 23 +++----- bacula/src/cats/sql_find.c | 3 +- bacula/src/cats/sql_get.c | 18 +++--- bacula/src/cats/sql_update.c | 2 +- bacula/src/stored/btape.c | 2 +- bacula/src/tools/dbcheck.c | 103 ++++++++++++++++------------------- bacula/src/tools/testfind.c | 20 +++++++ bacula/src/version.h | 4 +- 12 files changed, 108 insertions(+), 188 deletions(-) diff --git a/bacula/ChangeLog b/bacula/ChangeLog index fbb6ade4a8..e9c2a77e65 100644 --- a/bacula/ChangeLog +++ b/bacula/ChangeLog @@ -1,6 +1,7 @@ + ============================================================================= -2003-01-05 Version 1.28a released +2003-01-05 Version 1.28b released - Corrected a typo of working_directory in bacula-dir.conf reported by James MacLean. - Fixed the fact that path and filenames in some cases were not diff --git a/bacula/ReleaseNotes b/bacula/ReleaseNotes index d1b625f6d8..7cd5d3ef90 100644 --- a/bacula/ReleaseNotes +++ b/bacula/ReleaseNotes @@ -1,44 +1,10 @@ - Release Notes for Bacula 1.28 + Release Notes for Bacula 1.29 Bacula code: Total files = 228 Total lines = 62,042 (*.h *.c *.in) Major Changes this Release: -- Bare Metal Recovery mostly automated for Linux systems and - partially automated for Solaris -- Optimized restore, forward spaces to correct tape file and - stops reading the archive when all files are restored. -- Schedule permits specification of 1st, 2nd, ... week e.g. - 1st Sun ... Other Changes this Release: -- Allow backup of raw partitions. -- Fixed Restore options (never,ifnewer, ...). They now work. -- Added a --enable-client-only option that will cause "make" to only - build the File daemon and the libraries it needs. -- Documented how to save the Catalog database (with included script). -- Faster Storage daemon initialization (using pthreads) -- Unlimited devices in Storage daemon (previously hard coded to 20). -- Fixed bug that recognized only the last option specified on Include record. -- Easier building of statically linked version of Bacula. -- Fixed pruning bug. -- Fixes for VolUseDuration (problems putting it in DB). -- Fix for / specified as Where. -- Enhanced WriteBootstrap (more precise parameters -- e.g. volfile, ...) -- Improved handling of Differential jobs (now on par with Incremental) -- User supplied script in examples directory for backing up AFS file system. -- User supplied script for Sun-desktop autoloader (examples/devices). -- Added configure options for building static versions of the - daemons and the console program. -- New bidirectional timed pipe mechanism for running child processes - permits better error messages. -- Removed last restrictions on filename length for catalog items. Previous - maximum was 500 chars for path and 500 chars for filename. Now unlimited. -- Improvements all over the code to prevent string overflows and to ensure - that copied strings are properly terminated. -- Improved handling of 64 bit integers especially concerning the DB. -- Database now contains accurate tape file/block positions, and many other - DB items are more correctly handled. -- Improved bootstrap record generation (more detail). Items to note: - Nothing in particular. diff --git a/bacula/kernstodo b/bacula/kernstodo index a2eb615de5..10d3d4c77d 100644 --- a/bacula/kernstodo +++ b/bacula/kernstodo @@ -1,5 +1,5 @@ Kern's ToDo List - 12 January 2003 + 14 January 2003 Documentation to do: (a little bit at a time) - Document running a test version. @@ -7,34 +7,30 @@ Documentation to do: (a little bit at a time) - Document static linking - Document fixing tape file after crash (update Media set VolFiles=xx where MediaId=yy; update status to append). +- Document fifo and | and < Testing to do: (painful) - that console command line options work - blocksize recognition code. -- test fifo code -- test and fix < code and | code. For 1.29 release: - Add include list to end of chain in findlib - Look into Pruning/purging problems or why there seem to be so many files listed each night. -- Check out DB errors seen during recover. -- Write Unix emulator for Windows. - Why is catreq.c:111 Find vol called twice for a job? - Fix cancel in find_one -- need jcr. - Cancel does not work for restore in FD. - Write SetJobStatus() function so cancel status not lost. - Find out why Full saves run slower and slower (hashing?) -- Make 1.28c release +- Make 1.28c release ??? - Rewrite find_one.c to use only pool_memory instead of alloca and malloc. - Make sure btraceback goes into /sbin not sysconf directory. - Get correct error status from run_program or open_bpipe(). - Figure out how to allow multiple simultaneous file Volumes on a single device. -- Why are save/restore of device different sizes (sparse?) - Yup! Fix it. +- Why are save/restore of device different sizes (sparse?) Yup! Fix it. - Implement some why for the Console to dynamically create a job. - Restore to a particular time -- e.g. before date, after date. @@ -123,6 +119,7 @@ For 1.29 release: - Implement LabelTemplate (at least first cut). - Implement script driven addition of File daemon to config files. - Think about how to make Bacula work better with File (non-tape) archives. +- Write Unix emulator for Windows. - Implement new serialize subroutines send(socket, "string", &Vol, "uint32", &i, NULL) @@ -667,63 +664,3 @@ Results: ==================================== Done: (see kernsdone for more) -- Add EOM records? No, not at this time. The current system works and - above all is simple. -- Add VolumeUseDuration and MaximumVolumeJobs to Pool db record and - to Media db record. -- Add VOLUME_CAT_INFO to the EOS tape record (as - well as to the EOD record). -- No, not at this time. -- Put MaximumVolumeSize in Director (MaximumVolumeJobs, MaximumVolumeFiles, - MaximumFileSize). -- Enhance schedule to have 1stSat, ... -- Make sure catalog doesn't keep growing. -- On I/O error, write EOF, then try to write again ? No, keep it simple. -- Figure out how compress everything except .gz,... files. - Implement FileOptions. -- Put Bacula version somewhere in Job stream, probably Start Session Labels. -- Fix start/end blocks for File devices -- Make Job err if WriteBootstrap fails. -- Test that mod of restore options works. -- Test that week position schedule code works. -- Make BSR accept count (total files to be restored). -- Add code to fast seek to proper place on tape/file when doing Restore. -- Replace popen() and pclose() -- fail safe and timeout, no SIG dep. -- Add code to put VolFile in bsr for restore command. -- Volumes can be listed multiple times in Restore volume list. -- Add watchdog timeout for child processes start_child_timer() - end_child_timer(); -- Get rid of bscan.c:534 error message (one time only). -- Print some statistics when get EOF on device in bscan -- feedback - to let user know it is working. -- DateWritten field on tape may be wrong. -- Ensure that restore of differential jobs works (check SQL). -- Count number of ignored messages in bscan and print when first SOS is found. -- Test that EndFile/Block are correctly updated at end of tape - (in view of new block reading code). -- Test watchdog child timer code. -- Test new BSR code (mostly done). -- Work more on how to to a Bacula restore beginning with - just a Bacula tape and a boot floppy (bare metal recovery). -- Restore options (overwrite, overwrite if older, - overwrite if newer, never overwrite, ...) -- Fill all fields in Vol/Job Header -- ensure that everything - needed is written to tape. Think about restore to Catalog - from tape. Client record needs improving. -- Implement ./configure --with-client-only -- Finish up static linking -- that restore options work in FD -- Backup of raw partitions -- Document nofollow flag -- Make sure restore options are documented -- Should we dump a SOS when starting a new tape? (no -- too complicated) -- Fix gethostbyname() to use gethostbyname_r() -- no added mutex -- Add HOST to Volume label. -- Need to save contents of FileSet to tape? -- no not now -- Fix db_get_fileset in cats/sql_get.c for multiple records. -- Fix catalog filename truncation in sql_get and sql_create. Use - only a single filename split routine. -- Figure out how to save the catalog (possibly a special FileSet). -- Figure out how to restore the catalog. -- Make sure differential handled correctly -- Look at ua_prune.c in detail. Why did JobType work at all?????? -- Test size of hash table needed in find_one using testfind (about 1300). diff --git a/bacula/src/cats/sql.c b/bacula/src/cats/sql.c index 131887723d..7862f436f1 100644 --- a/bacula/src/cats/sql.c +++ b/bacula/src/cats/sql.c @@ -254,6 +254,11 @@ void db_end_transaction(B_DB *mdb) #endif } +/* + * Given a full filename, split it into its path + * and filename parts. They are returned in pool memory + * in the mdb structure. + */ void split_path_and_filename(B_DB *mdb, char *fname) { char *p, *f; @@ -283,7 +288,7 @@ void split_path_and_filename(B_DB *mdb, char *fname) mdb->fnl = p - f; if (mdb->fnl > 0) { mdb->fname = check_pool_memory_size(mdb->fname, mdb->fnl+1); - strncpy(mdb->fname, f, mdb->fnl); /* copy filename */ + memcpy(mdb->fname, f, mdb->fnl); /* copy filename */ mdb->fname[mdb->fnl] = 0; } else { mdb->fname[0] = ' '; /* blank filename */ @@ -294,7 +299,7 @@ void split_path_and_filename(B_DB *mdb, char *fname) mdb->pnl = f - fname; if (mdb->pnl > 0) { mdb->path = check_pool_memory_size(mdb->path, mdb->pnl+1); - strncpy(mdb->path, fname, mdb->pnl); + memcpy(mdb->path, fname, mdb->pnl); mdb->path[mdb->pnl] = 0; } else { Mmsg1(&mdb->errmsg, _("Path length is zero. File=%s\n"), fname); diff --git a/bacula/src/cats/sql_create.c b/bacula/src/cats/sql_create.c index 63c5e4ed12..e30ed500b9 100644 --- a/bacula/src/cats/sql_create.c +++ b/bacula/src/cats/sql_create.c @@ -26,8 +26,6 @@ */ -/* *****FIXME**** fix fixed length of select_cmd[] and insert_cmd[] */ - /* The following is necessary so that we do not include * the dummy external definition of DB. */ @@ -525,22 +523,20 @@ static int db_create_path_record(B_DB *mdb, ATTR_DBR *ar) if (mdb->cached_path_id != 0 && mdb->cached_path_len == mdb->pnl && strcmp(mdb->cached_path, mdb->path) == 0) { ar->PathId = mdb->cached_path_id; - ASSERT(ar->PathId); return 1; } Mmsg(&mdb->cmd, "SELECT PathId FROM Path WHERE Path='%s'", mdb->esc_name); if (QUERY_DB(mdb, mdb->cmd)) { - mdb->num_rows = sql_num_rows(mdb); - if (mdb->num_rows > 1) { char ed1[30]; - Mmsg2(&mdb->errmsg, _("More than one Path!: %s for Path=%s\n"), + Mmsg2(&mdb->errmsg, _("More than one Path!: %s for path: %s\n"), edit_uint64(mdb->num_rows, ed1), mdb->path); - Jmsg(mdb->jcr, M_ERROR, 0, "%s", mdb->errmsg); + Jmsg(mdb->jcr, M_WARNING, 0, "%s", mdb->errmsg); } + /* Even if there are multiple paths, take the first one */ if (mdb->num_rows >= 1) { if ((row = sql_fetch_row(mdb)) == NULL) { Mmsg1(&mdb->errmsg, _("error fetching row: %s\n"), sql_strerror(mdb)); @@ -579,12 +575,11 @@ static int db_create_path_record(B_DB *mdb, ATTR_DBR *ar) } /* Cache path */ - if (ar->PathId != mdb->cached_path_id) { + if (stat && ar->PathId != mdb->cached_path_id) { mdb->cached_path_id = ar->PathId; mdb->cached_path_len = mdb->pnl; pm_strcpy(&mdb->cached_path, mdb->path); } - ASSERT(ar->PathId); return stat; } @@ -602,13 +597,14 @@ static int db_create_filename_record(B_DB *mdb, ATTR_DBR *ar) if (QUERY_DB(mdb, mdb->cmd)) { mdb->num_rows = sql_num_rows(mdb); if (mdb->num_rows > 1) { - Mmsg2(&mdb->errmsg, _("More than one Filename!: %d File=%s\n"), - (int)(mdb->num_rows), mdb->fname); - Jmsg(mdb->jcr, M_ERROR, 0, "%s", mdb->errmsg); + char ed1[30]; + Mmsg2(&mdb->errmsg, _("More than one Filename! %s for file: %s\n"), + edit_uint64(mdb->num_rows, ed1), mdb->fname); + Jmsg(mdb->jcr, M_WARNING, 0, "%s", mdb->errmsg); } if (mdb->num_rows >= 1) { if ((row = sql_fetch_row(mdb)) == NULL) { - Mmsg2(&mdb->errmsg, _("error fetching row for file=%s: ERR=%s\n"), + Mmsg2(&mdb->errmsg, _("Error fetching row for file=%s: ERR=%s\n"), mdb->fname, sql_strerror(mdb)); Jmsg(mdb->jcr, M_ERROR, 0, "%s", mdb->errmsg); ar->FilenameId = 0; @@ -631,7 +627,6 @@ static int db_create_filename_record(B_DB *mdb, ATTR_DBR *ar) } else { ar->FilenameId = sql_insert_id(mdb); } - return ar->FilenameId > 0; } diff --git a/bacula/src/cats/sql_find.c b/bacula/src/cats/sql_find.c index c8ebb262ee..85b8b1c2e6 100644 --- a/bacula/src/cats/sql_find.c +++ b/bacula/src/cats/sql_find.c @@ -11,7 +11,7 @@ */ /* - Copyright (C) 2000, 2001, 2002 Kern Sibbald and John Walker + Copyright (C) 2000-2003 Kern Sibbald and John Walker This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as @@ -30,7 +30,6 @@ */ -/* *****FIXME**** fix fixed length of select_cmd[] and insert_cmd[] */ /* The following is necessary so that we do not include * the dummy external definition of DB. diff --git a/bacula/src/cats/sql_get.c b/bacula/src/cats/sql_get.c index da552c315a..886817b28c 100644 --- a/bacula/src/cats/sql_get.c +++ b/bacula/src/cats/sql_get.c @@ -29,7 +29,6 @@ */ -/* *****FIXME**** fix fixed length of select_cmd[] and insert_cmd[] */ /* The following is necessary so that we do not include * the dummy external definition of DB. @@ -159,12 +158,15 @@ static int db_get_filename_record(B_DB *mdb) Mmsg(&mdb->cmd, "SELECT FilenameId FROM Filename WHERE Name='%s'", mdb->esc_name); if (QUERY_DB(mdb, mdb->cmd)) { + char ed1[30]; mdb->num_rows = sql_num_rows(mdb); if (mdb->num_rows > 1) { - Mmsg1(&mdb->errmsg, _("More than one Filename!: %d\n"), (int)(mdb->num_rows)); - } + Mmsg2(&mdb->errmsg, _("More than one Filename!: %s for file: %s\n"), + edit_uint64(mdb->num_rows, ed1), mdb->fname); + Jmsg(mdb->jcr, M_WARNING, 0, "%s", mdb->errmsg); + } if (mdb->num_rows >= 1) { if ((row = sql_fetch_row(mdb)) == NULL) { Mmsg1(&mdb->errmsg, _("error fetching row: %s\n"), sql_strerror(mdb)); @@ -211,10 +213,12 @@ static int db_get_path_record(B_DB *mdb) mdb->num_rows = sql_num_rows(mdb); if (mdb->num_rows > 1) { - Mmsg1(&mdb->errmsg, _("More than one Path!: %s\n"), - edit_uint64(mdb->num_rows, ed1)); - Jmsg(mdb->jcr, M_ERROR, 0, "%s", mdb->errmsg); - } else if (mdb->num_rows == 1) { + Mmsg2(&mdb->errmsg, _("More than one Path!: %s for path: %s\n"), + edit_uint64(mdb->num_rows, ed1), mdb->path); + Jmsg(mdb->jcr, M_WARNING, 0, "%s", mdb->errmsg); + } + /* Even if there are multiple paths, take the first one */ + if (mdb->num_rows >= 1) { if ((row = sql_fetch_row(mdb)) == NULL) { Mmsg1(&mdb->errmsg, _("error fetching row: %s\n"), sql_strerror(mdb)); } else { diff --git a/bacula/src/cats/sql_update.c b/bacula/src/cats/sql_update.c index 56822936d7..e2c3b36c93 100644 --- a/bacula/src/cats/sql_update.c +++ b/bacula/src/cats/sql_update.c @@ -7,7 +7,7 @@ */ /* - Copyright (C) 2000, 2001, 2002 Kern Sibbald and John Walker + Copyright (C) 2000-2003 Kern Sibbald and John Walker This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as diff --git a/bacula/src/stored/btape.c b/bacula/src/stored/btape.c index ce91e5cd45..4d5c6cf108 100644 --- a/bacula/src/stored/btape.c +++ b/bacula/src/stored/btape.c @@ -1437,7 +1437,6 @@ static struct cmdstruct commands[] = { {"cap", capcmd, "list device capabilities"}, {"clear", clearcmd, "clear tape errors"}, {"eod", eodcmd, "go to end of Bacula data for append"}, - {"test", testcmd, "General test Bacula tape functions"}, {"eom", eomcmd, "go to the physical end of medium"}, {"fill", fillcmd, "fill tape, write onto second volume"}, {"unfill", unfillcmd, "read filled tape"}, @@ -1453,6 +1452,7 @@ static struct cmdstruct commands[] = { {"rewind", rewindcmd, "rewind the tape"}, {"scan", scancmd, "read tape block by block to EOT and report"}, {"status", statcmd, "print tape status"}, + {"test", testcmd, "General test Bacula tape functions"}, {"weof", weofcmd, "write an EOF on the tape"}, {"wr", wrcmd, "write a single record of 2048 bytes"}, }; diff --git a/bacula/src/tools/dbcheck.c b/bacula/src/tools/dbcheck.c index c37f0c7ac0..b1397bc109 100644 --- a/bacula/src/tools/dbcheck.c +++ b/bacula/src/tools/dbcheck.c @@ -9,7 +9,7 @@ * */ /* - Copyright (C) 2000, 2001, 2002 Kern Sibbald and John Walker + Copyright (C) 2000-2003 Kern Sibbald and John Walker This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as @@ -75,6 +75,7 @@ static void eliminate_orphaned_path_records(); static void eliminate_orphaned_filename_records(); static void eliminate_orphaned_fileset_records(); static void do_interactive_mode(); +static int yes_no(char *prompt); static void usage() @@ -188,10 +189,13 @@ static void do_interactive_mode() char *cmd; printf("Hello, this is the database check/correct program.\n\ -Please select the fuction you want to perform.\n"); +Modify database is %s. Verbose is %s.\n\ +Please select the fuction you want to perform.\n", + fix?"On":"Off", verbose?"On":"Off"); while (!quit) { - printf(_("\n\ + if (fix) { + printf(_("\n\ 1) Toggle modify database flag\n\ 2) Toggle verbose flag\n\ 3) Eliminate duplicate Filename records\n\ @@ -203,6 +207,20 @@ Please select the fuction you want to perform.\n"); 9) Eliminate orphaned FileSet records\n\ 10) All (3-9)\n\ 11) Quit\n")); + } else { + printf(_("\n\ + 1) Toggle modify database flag\n\ + 2) Toggle verbose flag\n\ + 3) Check for duplicate Filename records\n\ + 4) Check for duplicate Path records\n\ + 5) Check for orphaned Jobmedia records\n\ + 6) Check for orphaned File records\n\ + 7) Check for orphaned Path records\n\ + 8) Check for orphaned Filename records\n\ + 9) Check for orphaned FileSet records\n\ + 10) All (3-9)\n\ + 11) Quit\n")); + } cmd = get_cmd(_("Select function number: ")); if (cmd) { @@ -412,15 +430,11 @@ static void free_name_list(NAME_LIST *name_list) static void eliminate_duplicate_filenames() { char *query; + char esc_name[5000]; printf("Checking for duplicate Filename entries.\n"); /* Make list of duplicated names */ -#ifdef really_needed_for_HAVE_SQLITE - query = "SELECT Name FROM (SELECT COUNT(Name) as Count,Name from Filename " - "GROUP BY Name) WHERE Count > 1"; -#endif - query = "SELECT Name,count(Name) as Count FROM Filename GROUP BY Name " "HAVING Count > 1"; @@ -428,15 +442,18 @@ static void eliminate_duplicate_filenames() exit(1); } printf("Found %d duplicate Filename records.\n", name_list.num_ids); - if (verbose) { + if (verbose && yes_no("Print the list? (yes/no): ")) { print_name_list(&name_list); } if (fix) { /* Loop through list of duplicate names */ for (int i=0; i 1"; -#endif query = "SELECT Path,count(Path) as Count FROM Path " "GROUP BY Path HAVING Count > 1"; @@ -473,15 +487,18 @@ static void eliminate_duplicate_paths() exit(1); } printf("Found %d duplicate Path records.\n", name_list.num_ids); - if (verbose) { + if (verbose && yes_no("Print them? (yes/no): ")) { print_name_list(&name_list); } if (fix) { /* Loop through list of duplicate names */ for (int i=0; i