From 96cf382bdff2d9b83a46bddc8367f0b8706d1b5f Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Mon, 23 Feb 2004 16:10:04 +0000 Subject: [PATCH] Update projects + attempt to fix state file on Win32 git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@1076 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/projects | 201 +++++++++++++++++++++-------- bacula/src/filed/filed.c | 4 +- bacula/src/filed/win32/winmain.cpp | 8 +- bacula/src/lib/bsys.c | 9 +- bacula/src/lib/btime.c | 8 +- bacula/src/win32/compat/compat.cpp | 19 ++- bacula/src/win32/compat/compat.h | 70 +++++----- 7 files changed, 206 insertions(+), 113 deletions(-) diff --git a/bacula/projects b/bacula/projects index 8489863a0f..cc79273c48 100644 --- a/bacula/projects +++ b/bacula/projects @@ -1,15 +1,7 @@ Projects: - Bacula Projects Roadmap - 18 February 2004 - -Completed items from last year's list: -Item 1: Multiple simultaneous Jobs. (done) -Item 3: Write the bscan program -- also write a bcopy program (done). -Item 5: Implement Label templates (done). -Item 6: Write a regression script (done) -Item 9: Add SSL to daemon communications (For now, implement with stunnel) -Item 10: Define definitive tape format (done) + Bacula Projects Roadmap + 22 February 2004 Item 1: Implement Base jobs. @@ -17,28 +9,40 @@ Item 1: Implement Base jobs. will want the FileSet to contain only files that are unlikely to change in the future (i.e. a snapshot of most of your system after installing it). After the base job has been run, - when you are doing a Full save, you can specify to exclude - all files saved by the base job that have not been modified. - - Why: This is something none of the competition does, as far as we know + when you are doing a Full save, you specify one or more + Base jobs to be used. All files that have been backed up in + the Base job/jobs but not modified will then be excluded from + the backup. During a restore, the Base jobs will be + automatically pulled in where necessary. + + Why: This is something none of the competition does, as far as we +know (except BackupPC, which is a Perl program that saves to disk only). It is big win for the user, it makes Bacula stand out as offering a unique optimization that immediately saves time - and money. + and money. Basically, imagine that you have 100 nearly +identical + Windows or Linux machine containing the OS and user files. + Now for the OS part, a Base job will be backed up once, and + rather than making 100 copies of the OS, there will be only + one. If one or more of the systems have some files updated, + no problem, they will be automatically restored. - Notes: Big savings in tape usage. Will require more resources because - the DIR must send FD a list of files/attribs, and the FD must - search the list and compare it for each file to be saved. + Notes: Huge savings in tape usage even for a single machine. Will + require more resources because the DIR must send FD a list of + files/attribs, and the FD must search the list and compare it + for each file to be saved. -Item 2: Make the Storage daemon use intermediate file storage to buffer data - or Data Spooling. - What: If data is coming into the SD too fast, buffer it to - disk if the user has configured this option, so that tape - shuttling or shoe-shine can be reduced. +Item 2: Job Data Spooling. - Why: This would be a nice project and is the most requested feature. - Even though you may finish a client job quicker by spilling to + What: Make the Storage daemon use intermediate file storage to +buffer + the data to disk before writing it to the tape. + + Why: This would be a nice project and is the most requested +feature. + Even though you may finish a client job quicker by spooling to disk, you still have to eventually get it onto tape. If intermediate disk buffering allows us to improve write bandwidth to tape, it may make sense. In addition, you can @@ -46,9 +50,12 @@ Item 2: Make the Storage daemon use intermediate file storage to buffer data data can be written one job at a time to the tape at full tape speed. This keeps the tape running smoothly and prevents blocks from different simultaneous jobs from being intermixed - on the tape, which is very ineffficient for restores. + on the tape, which is very inefficient for restores. + + Notes: Need multiple spool directories. Should possibly be able + to spool by Job type, ... Possibly need high and low spool + data levels. - Notes: Item 3: GUI for interactive restore Item 4: GUI for interactive backup @@ -61,7 +68,8 @@ Item 4: GUI for interactive backup Why: Ease of use -- especially for the end user. - Notes: Rather than implementing in Gtk, we probably should go directly + Notes: Rather than implementing in Gtk, we probably should go +directly for a Browser implementation, even if doing so meant the capability wouldn't be available until much later. Not only is there the question of Windows sites, most @@ -72,23 +80,8 @@ Item 4: GUI for interactive backup own files will in most cases be on a Windows machine running Exploder. -Item 5: Implement data encryption (as opposed to communications - encryption) - - What: Currently the data that is stored on the Volume is not - encrypted. For confidentiality, encryption of data at - the File daemon level is essential. Note, communications - encryption encrypts the data when leaving the File daemon, - then decrypts the data on entry to the Storage daemon. - Data encryption encrypts the data in the File daemon and - decrypts the data in the File daemon during a restore. - Why: Large sites require this. - - Notes: The only algorithm that is needed is AES. - http://csrc.nist.gov/CryptoToolkit/aes/ - -Item 6: Implement a Migration job type that will move the job +Item 5: Implement a Migration job type that will move the job data from one device to another. What: The ability to copy, move, or archive data that is on a @@ -96,10 +89,12 @@ Item 6: Implement a Migration job type that will move the job Why: An ISP might want to backup to disk, but after 30 days migrate the data to tape backup and delete it from disk. - Bacula should be able to handle this automatically. It needs to + Bacula should be able to handle this automatically. It needs +to know what was put where, and when, and what to migrate -- it is a bit like retention periods. Doing so would allow space to - be freed up for current backups while maintaining older data on + be freed up for current backups while maintaining older data +on tape drives. Notes: Migration could be triggered by: @@ -110,7 +105,96 @@ Item 6: Implement a Migration job type that will move the job Lowwater mark -Item 7: New daemon communication protocol. +Item 6: Embedded Perl Scripting (precursor to 7). + + What: On a configuration parameter, embed the Perl language in + Bacula. + + Why: The embedded Perl scripting can be called to implement + Events such as "Volume Name needed", "End of Tape", + "Tape at x% of rated capacity", "Job started", + "Job Ended", "Job error", ... + + Notes: This needs Events. + + +Item 7: Implement Events (requires 6). + + What: When a particular user defined Event occurs, call the + embedded Perl interpreter. + + Why: This will provide the ultimate in user customization for + Bacula. Almost anything imaginable can be done if Events + are called at the appropriate place. + + Notes: There is a certain amount of work to be done on how + the user defines or "registers" events. + + +Item 8: Multiple Storage Devices for a Single Job + + What: Allow any Job to use more than one Storage device. + + Why: With two devices, for example, the second device could + have the next backup tape pre-mounted reducing operator + intervention in the middle of the night. + + +Item 9: Backup a Single Job Simultaneously to Multiple Storage + Devices + + What: Make two copies of the backup data at the same time. + + Why: Large shops typically do this and then take one set of + backups off-site. Some design work it needed in how to + specify the type of backup (backup, archive, ...) for each + Device. + + +Item 10: Break the one-to-one Relationship between a Job and a + Specific Storage Device (or Devices if #10 is implemented). + + What: Allow a Job to simply specify one or more MediaType, and the + Storage daemon will select a device for it. In fact, the user + should be able to specify one or more MediaType, Storage + daemon, and/or device to be used. + + Why: To allow more flexibility in large shops that have multiple + drives and/or multiple drives of different types. + + +Item 11: Add Regular Expression Matching and Plug-ins to the + FileSet Include statements. + + What: Allow users to specify wild-card and/or regular expressions + to be matched in both the Include and Exclude directives + in a FileSet. At the same time, allow users to define plug-ins + to be called (based on regular expression/wild-card matching). + + Why: This would give the users the ultimate ability to control how + files are backed up/restored. A user could write a plug-in + knows how to backup his Oracle database without + stopping/starting it, for example. + + +Item 12: Implement data encryption (as opposed to communications + encryption) + + What: Currently the data that is stored on the Volume is not + encrypted. For confidentiality, encryption of data at + the File daemon level is essential. Note, communications + encryption encrypts the data when leaving the File daemon, + then decrypts the data on entry to the Storage daemon. + Data encryption encrypts the data in the File daemon and + decrypts the data in the File daemon during a restore. + + Why: Large sites require this. + + Notes: The only algorithm that is needed is AES. + http://csrc.nist.gov/CryptoToolkit/aes/ + + +Item 13: New daemon communication protocol. What: The current daemon to daemon protocol is basically an ASCII printf() and sending the buffer. On the receiving end, the @@ -127,13 +211,16 @@ Item 7: New daemon communication protocol. Notes: These are internal improvements in the interest of the long-term stability and evolution of the program. On the one hand, the sooner they're done, the less code we have to rip - up when the time comes to install them. On the other hand, they - don't bring an immediately perceptible benefit to potential - users. - -To be documented: -Embedded Perl Scripting -Implement events -Multiple Storage devices for a single job -Write to more than one device simultaneously -Break the one-to-one relation between Storage and device + up when the time comes to install them. On the other hand, + they don't bring an immediately perceptible benefit to + potential users. + + +Completed items from last year's list: +Item 1: Multiple simultaneous Jobs. (done) +Item 3: Write the bscan program -- also write a bcopy program (done). +Item 5: Implement Label templates (done). +Item 6: Write a regression script (done) +Item 9: Add SSL to daemon communications (For now, implement with +stunnel) +Item 10: Define definitive tape format (done) diff --git a/bacula/src/filed/filed.c b/bacula/src/filed/filed.c index bd8fd35704..3a7b5bb85b 100644 --- a/bacula/src/filed/filed.c +++ b/bacula/src/filed/filed.c @@ -248,14 +248,14 @@ Without that I don't know who I am :-(\n"), configfile); void terminate_filed(int sig) { + write_state_file(me->working_directory, "bacula-fd", me->FDport); + delete_pid_file(me->pid_directory, "bacula-fd", me->FDport); if (configfile != NULL) { free(configfile); } if (debug_level > 5) { print_memory_pool_stats(); } - write_state_file(me->working_directory, "bacula-fd", me->FDport); - delete_pid_file(me->pid_directory, "bacula-fd", me->FDport); free_config_resources(); term_msg(); stop_watchdog(); diff --git a/bacula/src/filed/win32/winmain.cpp b/bacula/src/filed/win32/winmain.cpp index b839b961f4..1b50502c3a 100755 --- a/bacula/src/filed/win32/winmain.cpp +++ b/bacula/src/filed/win32/winmain.cpp @@ -36,9 +36,10 @@ #include "../../findlib/winapi.h" extern int BaculaMain(int argc, char *argv[]); -extern int terminate_filed(int sig); +extern void terminate_filed(int sig); extern DWORD g_error; extern BOOL ReportStatus(DWORD state, DWORD exitcode, DWORD waithint); +extern void d_msg(const char *, int, int, const char *, ...); /* Globals */ HINSTANCE hAppInstance; @@ -299,7 +300,10 @@ void *Main_Msg_Loop(LPVOID lpwThreadParam) // Tell the service manager that we've stopped. ReportStatus(SERVICE_STOPPED, g_error, 0); - } + } + terminate_filed(0); + + /* Should not get here */ pthread_kill(main_tid, SIGTERM); /* ask main thread to terminate */ sleep(1); kill(main_pid, SIGTERM); /* ask main thread to terminate */ diff --git a/bacula/src/lib/bsys.c b/bacula/src/lib/bsys.c index 96eab83660..41312daf68 100644 --- a/bacula/src/lib/bsys.c +++ b/bacula/src/lib/bsys.c @@ -353,13 +353,13 @@ int delete_pid_file(char *dir, const char *progname, int port) struct s_state_hdr { char id[14]; int32_t version; - uint32_t last_jobs_addr; - uint32_t reserved[20]; + uint64_t last_jobs_addr; + uint64_t reserved[20]; }; static struct s_state_hdr state_hdr = { "Bacula State\n", - 1, + 2, 0 }; @@ -402,15 +402,18 @@ void write_state_file(char *dir, const char *progname, int port) Mmsg(&fname, "%s/%s.%d.state", dir, progname, port); /* Create new state file */ if ((sfd = open(mp_chr(fname), O_CREAT | O_TRUNC | O_WRONLY, 0640)) < 0) { + Dmsg2(000, _("Could not create state file. %s ERR=%s\n"), fname, strerror(errno)); Emsg2(M_ERROR, 0, _("Could not create state file. %s ERR=%s\n"), fname, strerror(errno)); goto bail_out; } if (write(sfd, &state_hdr, sizeof(state_hdr)) < 0) { + Dmsg1(000, "Write error: ERR=%s\n", strerror(errno)); goto bail_out; } state_hdr.last_jobs_addr = sizeof(state_hdr); state_hdr.reserved[0] = write_last_jobs_list(sfd, state_hdr.last_jobs_addr); if (lseek(sfd, 0, SEEK_SET) < 0) { + Dmsg1(000, "lseek error: ERR=%s\n", strerror(errno)); goto bail_out; } write(sfd, &state_hdr, sizeof(state_hdr)); diff --git a/bacula/src/lib/btime.c b/bacula/src/lib/btime.c index f5e3a99b2c..aa798b7797 100644 --- a/bacula/src/lib/btime.c +++ b/bacula/src/lib/btime.c @@ -44,7 +44,7 @@ /* Formatted time for user display: dd-Mon-yyyy hh:mm */ char *bstrftime(char *dt, int maxlen, utime_t tim) { - time_t ttime = tim; + time_t ttime = (time_t)tim; struct tm tm; /* ***FIXME**** the format and localtime_r() should be user configurable */ @@ -56,7 +56,7 @@ char *bstrftime(char *dt, int maxlen, utime_t tim) /* Formatted time for user display: dd-Mon-yy hh:mm (no century) */ char *bstrftime_nc(char *dt, int maxlen, utime_t tim) { - time_t ttime = tim; + time_t ttime = (time_t)tim; struct tm tm; /* ***FIXME**** the format and localtime_r() should be user configurable */ @@ -71,7 +71,7 @@ char *bstrftime_nc(char *dt, int maxlen, utime_t tim) /* Unix time to standard time string yyyy-mm-dd hh:mm:ss */ char *bstrutime(char *dt, int maxlen, utime_t tim) { - time_t ttime = tim; + time_t ttime = (time_t)tim; struct tm tm; localtime_r(&ttime, &tm); strftime(dt, maxlen, "%Y-%m-%d %H:%M:%S", &tm); @@ -311,7 +311,7 @@ void time_decode(ftime_t time, uint8_t *hour, uint8_t *minute, *minute = (uint8_t) ((ij / 60L) % 60L); *second = (uint8_t) (ij % 60L); if (second_fraction != NULL) { - *second_fraction = time - floor(time); + *second_fraction = (float32_t)(time - floor(time)); } } diff --git a/bacula/src/win32/compat/compat.cpp b/bacula/src/win32/compat/compat.cpp index a324579880..36b0fd5014 100644 --- a/bacula/src/win32/compat/compat.cpp +++ b/bacula/src/win32/compat/compat.cpp @@ -22,8 +22,8 @@ // Author : Christopher S. Hull // Created On : Sat Jan 31 15:55:00 2004 // Last Modified By: Christopher S. Hull -// Last Modified On: Sun Feb 22 16:08:54 2004 -// Update Count : 659 +// Last Modified On: Sun Feb 22 12:55:40 2004 +// Update Count : 634 // $Id$ #include @@ -37,7 +37,7 @@ extern DWORD g_platform_id; // from CYGWIN (should be diff between Jan 1 1601 and Jan 1 1970 #define WIN32_FILETIME_ADJUST 0x19DB1DED53E8000I64 -#define WIN32_FILETIME_SCALE 10000000 // 100ns/second +#define WIN32_FILETIME_SCALE 10000000 // 100ns/second extern "C" void cygwin_conv_to_win32_path(const char *name, char *win32_name) @@ -137,8 +137,8 @@ close(int fd) return 0; } -int -write(int fd, const void *data, int32_t len) +ssize_t +write(int fd, const void *data, size_t len) { BOOL status; DWORD bwrite; @@ -148,15 +148,14 @@ write(int fd, const void *data, int32_t len) } -int -read(int fd, void *data, int32_t len) +ssize_t +read(int fd, void *data, size_t len) { BOOL status; DWORD bread; status = ReadFile((HANDLE)fd, data, len, &bread, NULL); if (status) return bread; - return -1; } @@ -471,8 +470,8 @@ readlink(const char *, char *, int) return -1; } -int -lseek(int fd, int offset, int whence) +off_t +lseek(int fd, off_t offset, int whence) { DWORD method = 0; switch (whence) { diff --git a/bacula/src/win32/compat/compat.h b/bacula/src/win32/compat/compat.h index 9591f83035..9b6c562c49 100644 --- a/bacula/src/win32/compat/compat.h +++ b/bacula/src/win32/compat/compat.h @@ -21,8 +21,8 @@ * Author : Christopher S. Hull * Created On : Fri Jan 30 13:00:51 2004 * Last Modified By: Christopher S. Hull - * Last Modified On: Sun Feb 22 13:15:59 2004 - * Update Count : 207 + * Last Modified On: Mon Feb 9 12:08:07 2004 + * Update Count : 201 * $Id$ */ @@ -92,16 +92,17 @@ typedef UINT32 key_t; typedef UINT32 uid_t; typedef UINT32 gid_t; typedef UINT32 mode_t; -typedef UINT32 ssize_t; +typedef INT64 ssize_t; typedef void DIR; + struct dirent { - uint64_t d_ino; - uint32_t d_off; - uint16_t d_reclen; - char d_name[256]; + uint64_t d_ino; + uint32_t d_off; + uint16_t d_reclen; + char d_name[256]; }; #ifndef __cplusplus @@ -125,19 +126,19 @@ int gettimeofday(struct timeval *, struct timezone *); struct stat { - _dev_t st_dev; - uint64_t st_ino; - uint16_t st_mode; - int16_t st_nlink; - uint32_t st_uid; - uint32_t st_gid; - _dev_t st_rdev; - uint64_t st_size; - time_t st_atime; - time_t st_mtime; - time_t st_ctime; - uint32_t st_blksize; - uint64_t st_blocks; + _dev_t st_dev; + uint64_t st_ino; + uint16_t st_mode; + int16_t st_nlink; + uint32_t st_uid; + uint32_t st_gid; + _dev_t st_rdev; + uint64_t st_size; + time_t st_atime; + time_t st_mtime; + time_t st_ctime; + uint32_t st_blksize; + uint64_t st_blocks; }; #define S_IFMT 0170000 /* file type mask */ @@ -149,23 +150,23 @@ struct stat #define S_IWRITE 0000200 /* write permission, owner */ #define S_IEXEC 0000100 /* execute/search permission, owner */ -#define S_IRUSR S_IREAD -#define S_IWUSR S_IWRITE -#define S_IXUSR S_IEXEC +#define S_IRUSR S_IREAD +#define S_IWUSR S_IWRITE +#define S_IXUSR S_IEXEC -#define S_IRGRP 000040 +#define S_IRGRP 000040 #define S_IWGRP 000020 #define S_IXGRP 000010 -#define S_IROTH 00004 +#define S_IROTH 00004 #define S_IWOTH 00002 #define S_IXOTH 00001 -#define S_IRWXO 000007 -#define S_IRWXG 000070 -#define S_ISUID 004000 -#define S_ISGID 002000 -#define S_ISVTX 001000 +#define S_IRWXO 000007 +#define S_IRWXG 000070 +#define S_ISUID 004000 +#define S_ISGID 002000 +#define S_ISVTX 001000 #define S_ISREG(x) (((x) & S_IFREG) == S_IFREG) @@ -174,7 +175,7 @@ struct stat #define S_ISBLK(x) 0 #define S_ISFIFO(x) 0 #define S_ISSOCK(x) 0 -#define S_ISLNK(x) 0 +#define S_ISLNK(x) 0 #define SIGUSR2 9999 @@ -184,9 +185,9 @@ int chown(const char *, uid_t uid, gid_t gid); int chmod(const char *, mode_t mode); int utime(const char *filename, struct utimbuf *buf); int open(const char *, int, int); -int lseek(int, int, int); -int read(int fd, void *, int32_t nbytes); -int write(int fd, const void *, int32_t nbytes); +off_t lseek(int, off_t, int); +ssize_t read(int fd, void *, size_t nbytes); +ssize_t write(int fd, const void *, size_t nbytes); int close(int fd); int inet_aton(const char *cp, struct in_addr *inp); int kill(int pid, int signo); @@ -288,4 +289,3 @@ int win32_chdir(const char *buf); int WSA_Init(void); #endif /* __COMPAT_H_ */ - -- 2.39.5